Neural_Machine_Translation_tf_2L_8H_Loss_Changed.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Neural_Machine_Translation_tf-2L-8H-Loss-Changed.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#### Mounting Google Drive"
      ],
      "metadata": {
        "id": "0dqNGaFLKKAJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "bVenoImvJZs9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c852767f-5687-4193-97dd-677b4c90594f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd \"/content/drive/MyDrive/IASNLP\""
      ],
      "metadata": {
        "id": "iNpRDc1-KMj8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7b89ba6c-0e5c-47cd-b72b-0a635542fc65"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/IASNLP\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Importing Necessary Libraries"
      ],
      "metadata": {
        "id": "OlMuBUdIApMX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install sentencepiece\n",
        "!pip install sacrebleu\n",
        "!pip install tensorflow-gpu # ---> For GPU"
      ],
      "metadata": {
        "id": "Iwi6TDKsB0St",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c7b80d37-f233-4937-bb46-05a975c627a1"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 5.3 MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.96\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.1.0-py3-none-any.whl (92 kB)\n",
            "\u001b[K     |████████████████████████████████| 92 kB 4.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (0.8.9)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (2022.6.2)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.5-py2.py3-none-any.whl (16 kB)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.4.0-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (1.21.6)\n",
            "Installing collected packages: portalocker, colorama, sacrebleu\n",
            "Successfully installed colorama-0.4.5 portalocker-2.4.0 sacrebleu-2.1.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting tensorflow-gpu\n",
            "  Downloading tensorflow_gpu-2.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (511.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 511.7 MB 6.0 kB/s \n",
            "\u001b[?25hCollecting flatbuffers<2,>=1.12\n",
            "  Downloading flatbuffers-1.12-py2.py3-none-any.whl (15 kB)\n",
            "Collecting gast<=0.4.0,>=0.2.1\n",
            "  Downloading gast-0.4.0-py3-none-any.whl (9.8 kB)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.1.2)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.6.3)\n",
            "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (3.17.3)\n",
            "Collecting tensorflow-estimator<2.10.0,>=2.9.0rc0\n",
            "  Downloading tensorflow_estimator-2.9.0-py2.py3-none-any.whl (438 kB)\n",
            "\u001b[K     |████████████████████████████████| 438 kB 65.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.1.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.15.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (0.2.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (57.4.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (3.3.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (21.3)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.14.1)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.21.6)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (4.1.1)\n",
            "Collecting keras<2.10.0,>=2.9.0rc0\n",
            "  Downloading keras-2.9.0-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 47.7 MB/s \n",
            "\u001b[?25hCollecting tensorboard<2.10,>=2.9\n",
            "  Downloading tensorboard-2.9.1-py3-none-any.whl (5.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.8 MB 57.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (0.26.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.46.3)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (3.1.0)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (14.0.1)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-gpu) (1.1.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow-gpu) (0.37.1)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow-gpu) (1.5.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (1.0.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (1.35.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (3.3.7)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (0.6.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow-gpu) (1.8.1)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (4.2.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (4.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow-gpu) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow-gpu) (4.11.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow-gpu) (3.8.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow-gpu) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (2022.6.15)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow-gpu) (3.0.4)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow-gpu) (3.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->tensorflow-gpu) (3.0.9)\n",
            "Installing collected packages: tensorflow-estimator, tensorboard, keras, gast, flatbuffers, tensorflow-gpu\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.8.0\n",
            "    Uninstalling tensorflow-estimator-2.8.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.8.0\n",
            "  Attempting uninstall: tensorboard\n",
            "    Found existing installation: tensorboard 2.8.0\n",
            "    Uninstalling tensorboard-2.8.0:\n",
            "      Successfully uninstalled tensorboard-2.8.0\n",
            "  Attempting uninstall: keras\n",
            "    Found existing installation: keras 2.8.0\n",
            "    Uninstalling keras-2.8.0:\n",
            "      Successfully uninstalled keras-2.8.0\n",
            "  Attempting uninstall: gast\n",
            "    Found existing installation: gast 0.5.3\n",
            "    Uninstalling gast-0.5.3:\n",
            "      Successfully uninstalled gast-0.5.3\n",
            "  Attempting uninstall: flatbuffers\n",
            "    Found existing installation: flatbuffers 2.0\n",
            "    Uninstalling flatbuffers-2.0:\n",
            "      Successfully uninstalled flatbuffers-2.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.8.2+zzzcolab20220527125636 requires keras<2.9,>=2.8.0rc0, but you have keras 2.9.0 which is incompatible.\n",
            "tensorflow 2.8.2+zzzcolab20220527125636 requires tensorboard<2.9,>=2.8, but you have tensorboard 2.9.1 which is incompatible.\n",
            "tensorflow 2.8.2+zzzcolab20220527125636 requires tensorflow-estimator<2.9,>=2.8, but you have tensorflow-estimator 2.9.0 which is incompatible.\u001b[0m\n",
            "Successfully installed flatbuffers-1.12 gast-0.4.0 keras-2.9.0 tensorboard-2.9.1 tensorflow-estimator-2.9.0 tensorflow-gpu-2.9.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras import Sequential\n",
        "from tensorflow.keras.layers import Dense, Lambda, Layer, Embedding, LayerNormalization\n",
        "import sentencepiece as spm\n",
        "from sacrebleu.metrics import BLEU, CHRF, TER\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "from collections import Counter"
      ],
      "metadata": {
        "id": "DXSYYy2b-jMM"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading Data"
      ],
      "metadata": {
        "id": "ToLMSNAHBnvJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We load the `train_data`, `train_dev_data`, `test_val_data` and `test_data` as well as the byte-pair encoder tokenizer for English and Bengali i.e. `sp_en_bpe` and `s_ben_bpe`."
      ],
      "metadata": {
        "id": "zXIf-FL7IG7f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_data, train_dev_data = pd.read_csv(\"train_data.csv\")[['src', 'tgt']], pd.read_csv(\"train_dev.csv\")[['src', 'tgt']] \n",
        "test_val_data, test_data = pd.read_csv(\"test_val.csv\")[['src', 'tgt']], pd.read_csv(\"test_data.csv\")[['src', 'tgt']]\n",
        "sp_en_bpe, sp_ben_bpe = spm.SentencePieceProcessor(), spm.SentencePieceProcessor()\n",
        "sp_en_bpe.load('eng_bpe.model'); sp_ben_bpe.load('ben_bpe.model');"
      ],
      "metadata": {
        "id": "IXsCg4mBBeoo"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data Preprocessing"
      ],
      "metadata": {
        "id": "auf4TlPFhNtn"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We decide on some formats of the data and batches. We take the maximum tokens we can have per tokenized English and Bengali sentence to be `MAX_TOKENS = 60`. We use the vocabulary size of `32000`(i.e. `ENCODER_VOCAB = DECODER_VOCAB = 32000`). A batch size of `256` was choosen for our Mini-Batch Gradient Descent which is used to train our Model."
      ],
      "metadata": {
        "id": "3z7BUnn_G3WL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "MAX_TOKENS = 60\n",
        "VOCAB_SIZE = 32000\n",
        "BATCH_SIZE = 256\n",
        "BUFFER_SIZE = BATCH_SIZE*4"
      ],
      "metadata": {
        "id": "7jTuQSqOAybO"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The following two functions help in tokenizing an detokenizing the data based on our sentencepiece byte-pair encoding model."
      ],
      "metadata": {
        "id": "NlEW7GndbCfw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize(sentence, sp_model, add_bos = True, add_eos = True):\n",
        "    bos = [sp_model.bos_id()] if add_bos else []\n",
        "    eos = [sp_model.eos_id()] if add_eos else []\n",
        "    inputs = bos + sp_model.encode_as_ids(sentence) + eos\n",
        "    return np.reshape(np.array(inputs), [1, -1])\n",
        "def detokenize(tokenized, sp_model, is_bos = True, is_eos = True):\n",
        "    integers = np.squeeze(tokenized).tolist()\n",
        "    if is_eos:\n",
        "        return sp_model.DecodeIdsWithCheck(integers[int(is_bos):integers.index(sp_model.eos_id())])\n",
        "    else:\n",
        "        if sp_model.pad_id() in tokenized:\n",
        "            return sp_model.DecodeIdsWithCheck(integers[int(is_bos):integers.index(sp_model.pad_id())])\n",
        "        else:\n",
        "            return sp_model.DecodeIdsWithCheck(integers[int(is_bos):])"
      ],
      "metadata": {
        "id": "VC-6hav2BrJJ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have tokenized the data."
      ],
      "metadata": {
        "id": "5-76mwBnbuzp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Tokenization\n",
        "src_train_data_enc = [np.squeeze(tokenize(train_data['src'].iloc[i], sp_en_bpe)) for i in range(train_data.shape[0])]\n",
        "tgt_train_data_enc = [np.squeeze(tokenize(train_data['tgt'].iloc[i], sp_ben_bpe)) for i in range(train_data.shape[0])]\n",
        "src_train_dev_data_enc = [np.squeeze(tokenize(train_dev_data['src'].iloc[i], sp_en_bpe)) for i in range(train_dev_data.shape[0])]\n",
        "tgt_train_dev_data_enc = [np.squeeze(tokenize(train_dev_data['tgt'].iloc[i], sp_ben_bpe)) for i in range(train_dev_data.shape[0])]\n",
        "src_test_val_data_enc = [np.squeeze(tokenize(test_val_data['src'].iloc[i], sp_en_bpe)) for i in range(test_val_data.shape[0])]\n",
        "tgt_test_val_data_enc = [np.squeeze(tokenize(test_val_data['tgt'].iloc[i], sp_ben_bpe)) for i in range(test_val_data.shape[0])]"
      ],
      "metadata": {
        "id": "nT0QpepuDoFF"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We have padded the data with post-truncation and post padding upto MAX_TOKENS number of tokens."
      ],
      "metadata": {
        "id": "wdyEjbm5hWXg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Padding\n",
        "# The reason why we add MAX_TOKENS+1 after-pad length for Bengali is that when we form the tf.Dataset later, we will shift the target sentence once and will truncate the last token once\n",
        "# Hence, it will get adjusted\n",
        "train_src = tf.keras.preprocessing.sequence.pad_sequences(src_train_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "train_tgt = tf.keras.preprocessing.sequence.pad_sequences(tgt_train_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "train_dev_src = tf.keras.preprocessing.sequence.pad_sequences(src_train_dev_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "train_dev_tgt = tf.keras.preprocessing.sequence.pad_sequences(tgt_train_dev_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "test_val_src = tf.keras.preprocessing.sequence.pad_sequences(src_test_val_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "test_val_tgt = tf.keras.preprocessing.sequence.pad_sequences(tgt_test_val_data_enc, maxlen = MAX_TOKENS, padding='post', truncating='post')"
      ],
      "metadata": {
        "id": "NT63vC8PcK-2"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below, we discard all those pairs where the tokenized source or target sentences are larger than MAX_TOKENS number of tokens."
      ],
      "metadata": {
        "id": "FvxtnfKniMRA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Removing all pairs whose either the target sentence or source setence exceeds MAX_TOKENS number of tokens\n",
        "same_idx = list(set([i for i in range(train_src.shape[0]) if train_src[i][-1] == 1 or train_src[i][-1] == 0]).intersection(set([i for i in range(train_tgt.shape[0]) if train_tgt[i][-1] == 1 or train_tgt[i][-1] == 0])))\n",
        "train_src = train_src[same_idx]\n",
        "train_tgt = train_tgt[same_idx]\n",
        "same_idx = list(set([i for i in range(train_dev_src.shape[0]) if train_dev_src[i][-1] == 1 or train_dev_src[i][-1] == 0]).intersection(set([i for i in range(train_dev_tgt.shape[0]) if train_dev_tgt[i][-1] == 1 or train_dev_tgt[i][-1] == 0])))\n",
        "train_dev_src = train_dev_src[same_idx]\n",
        "train_dev_tgt = train_dev_tgt[same_idx]\n",
        "same_idx = list(set([i for i in range(test_val_src.shape[0]) if test_val_src[i][-1] == 1 or test_val_src[i][-1] == 0]).intersection(set([i for i in range(test_val_tgt.shape[0]) if test_val_tgt[i][-1] == 1 or test_val_tgt[i][-1] == 0])))\n",
        "test_val_src = test_val_src[same_idx]\n",
        "test_val_tgt = test_val_tgt[same_idx]"
      ],
      "metadata": {
        "id": "L6-cV6zpcO5e"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_labels = np.zeros(train_tgt.shape)\n",
        "train_labels[:,0:train_tgt.shape[1] -1] = train_tgt[:,1:]\n",
        "train_dev_labels = np.zeros(train_dev_tgt.shape)\n",
        "train_dev_labels[:,0:train_dev_tgt.shape[1] -1] = train_dev_tgt[:,1:]\n",
        "test_val_labels = np.zeros(test_val_tgt.shape)\n",
        "test_val_labels[:,0:test_val_tgt.shape[1] -1] = test_val_tgt[:,1:]"
      ],
      "metadata": {
        "id": "JGMvPvIxOx-0"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"*** Preprocessed Data ***\")\n",
        "print(\"Train Data Shape(Source): \", train_src.shape)\n",
        "print(\"Train Data Shape(Target): \", train_tgt.shape)\n",
        "print(\"Train Labels Shape(Target): \", train_labels.shape)\n",
        "print(\"Train Dev Data Shape(Source): \", train_dev_src.shape)\n",
        "print(\"Train Dev Data Shape(Target): \", train_dev_tgt.shape)\n",
        "print(\"Train Dev Labels Shape(Target): \", train_dev_labels.shape)\n",
        "print(\"Test Val Data Shape(Source): \", test_val_src.shape)\n",
        "print(\"Test Val Data Shape(Target): \", test_val_tgt.shape)\n",
        "print(\"Test Labels Shape(Target): \", test_val_labels.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rO5HD33XOGxf",
        "outputId": "bd274301-633e-40e4-a278-c33feca523ae"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "*** Preprocessed Data ***\n",
            "Train Data Shape(Source):  (154836, 60)\n",
            "Train Data Shape(Target):  (154836, 60)\n",
            "Train Labels Shape(Target):  (154836, 60)\n",
            "Train Dev Data Shape(Source):  (3649, 60)\n",
            "Train Dev Data Shape(Target):  (3649, 60)\n",
            "Train Dev Labels Shape(Target):  (3649, 60)\n",
            "Test Val Data Shape(Source):  (1194, 60)\n",
            "Test Val Data Shape(Target):  (1194, 60)\n",
            "Test Labels Shape(Target):  (1194, 60)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below we prepare the data in the form ready to feed to the Model."
      ],
      "metadata": {
        "id": "xtsMxbAYzePz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_data_prep = tf.data.Dataset.from_tensor_slices((train_src, train_tgt, train_labels))\n",
        "train_dev_data_prep = tf.data.Dataset.from_tensor_slices((train_dev_src, train_dev_tgt, train_dev_labels))\n",
        "test_val_data_prep = tf.data.Dataset.from_tensor_slices((test_val_src, test_val_tgt, test_val_labels))"
      ],
      "metadata": {
        "id": "_OkR5tbtyBQj"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Below, we can see how our data looks"
      ],
      "metadata": {
        "id": "jSGBKDhc2sc-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for encoder_inputs, decoder_inputs, labels in train_data_prep.take(1):\n",
        "    print(f'encoder_inputs.shape: {encoder_inputs.shape}')\n",
        "    print(f'decoder_inputs.shape: {decoder_inputs.shape}')\n",
        "    print(f\"labels.shape: {labels.shape}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ao1TX6GHzWn7",
        "outputId": "51d64474-5c88-4f3b-c710-3b9867a38829"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "encoder_inputs.shape: (60,)\n",
            "decoder_inputs.shape: (60,)\n",
            "labels.shape: (60,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model"
      ],
      "metadata": {
        "id": "zjXk6pt-28kL"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will use the Transformer Architechture with the following hyperparameters."
      ],
      "metadata": {
        "id": "a9UFHuFO3dHe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Transformer parameters\n",
        "d_model = 256 # 512 in the original paper\n",
        "d_k = 32 # 64 in the original paper\n",
        "d_v = 32 # 64 in the original paper\n",
        "n_heads = 8 # 8 in the original paper\n",
        "n_encoder_layers = 2 # 6 in the original paper\n",
        "n_decoder_layers = 2 # 6 in the original paper\n",
        "\n",
        "max_token_length = MAX_TOKENS # 512 in the original paper"
      ],
      "metadata": {
        "id": "v9ULcpgbJ9WP"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class SingleHeadAttention(Layer):\n",
        "  def __init__(self, input_shape=(3, -1, d_model), dropout=.0, masked=None):\n",
        "    super(SingleHeadAttention, self).__init__()\n",
        "    self.q = Dense(d_k, input_shape=(-1, d_model), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    self.normalize_q = Lambda(lambda x: x / np.sqrt(d_k))\n",
        "    self.k = Dense(d_k, input_shape=(-1, d_model), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    self.v = Dense(d_v, input_shape=(-1, d_model), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    self.dropout = dropout\n",
        "    self.masked = masked\n",
        "  \n",
        "  # Inputs: [query, key, value]\n",
        "  def call(self, inputs, training=None):\n",
        "    assert len(inputs) == 3\n",
        "    # We use a lambda layer to divide vector q by sqrt(d_k) according to the equation\n",
        "    q = self.normalize_q(self.q(inputs[0]))\n",
        "    k = self.k(inputs[1])\n",
        "    # The dimensionality of q is (batch_size, query_length, d_k) and that of k is (batch_size, key_length, d_k)\n",
        "    # So we will do a matrix multication by batch after transposing last 2 dimensions of k\n",
        "    # tf.shape(attn_weights) = (batch_size, query_length, key_length)\n",
        "    attn_weights = tf.matmul(q, tf.transpose(k, perm=[0,2,1]))\n",
        "    if self.masked: # Prevent future attentions in decoding self-attention\n",
        "      # Create a matrix where the strict upper triangle (not including main diagonal) is filled with -inf and 0 elsewhere\n",
        "      length = tf.shape(attn_weights)[-1]\n",
        "      #attn_mask = np.triu(tf.fill((length, length), -np.inf), k=1) # We need to use tensorflow functions instead of numpy\n",
        "      attn_mask = tf.fill((length, length), -np.inf)\n",
        "      attn_mask = tf.linalg.band_part(attn_mask, 0, -1) # Get upper triangle\n",
        "      attn_mask = tf.linalg.set_diag(attn_mask, tf.zeros((length))) # Set diagonal to zeros to avoid operations with infinity\n",
        "      # This matrix is added to the attention weights so all future attention will have -inf logits (0 after softmax)\n",
        "      attn_weights += attn_mask\n",
        "    # Softmax along the last dimension\n",
        "    attn_weights = tf.nn.softmax(attn_weights, axis=-1)\n",
        "    if training: # Attention dropout included in the original paper. This is possibly to encourage multihead diversity.\n",
        "      attn_weights = tf.nn.dropout(attn_weights, rate=self.dropout)\n",
        "    v = self.v(inputs[2])\n",
        "    return tf.matmul(attn_weights, v)\n",
        "\n",
        "class MultiHeadAttention_my(Layer):\n",
        "  def __init__(self, dropout=.0, masked=None):\n",
        "    super(MultiHeadAttention_my, self).__init__()\n",
        "    self.attn_heads = list()\n",
        "    for i in range(n_heads): \n",
        "      self.attn_heads.append(SingleHeadAttention(dropout=dropout, masked=masked))\n",
        "    self.linear = Dense(d_model, input_shape=(-1, n_heads * d_v), kernel_initializer='glorot_uniform', \n",
        "                   bias_initializer='glorot_uniform')\n",
        "    \n",
        "  def call(self, x, training=None):\n",
        "    attentions = [self.attn_heads[i](x, training=training) for i in range(n_heads)]\n",
        "    concatenated_attentions = tf.concat(attentions, axis=-1)\n",
        "    return self.linear(concatenated_attentions)\n",
        "\n",
        "class TransformerEncoder(Layer):\n",
        "  def __init__(self, dropout=.1, attention_dropout=.0, **kwargs):\n",
        "    super(TransformerEncoder, self).__init__(**kwargs)\n",
        "    self.dropout_rate = dropout\n",
        "    self.attention_dropout_rate = attention_dropout\n",
        "  def build(self, input_shape):\n",
        "    self.multihead_attention = MultiHeadAttention_my(dropout=self.attention_dropout_rate)\n",
        "    self.dropout1 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization1 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    \n",
        "    self.linear1 = Dense(input_shape[-1] * 4, input_shape=input_shape, activation='relu',\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.linear2 = Dense(input_shape[-1], input_shape=self.linear1.compute_output_shape(input_shape),\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.dropout2 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization2 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    super(TransformerEncoder, self).build(input_shape)\n",
        "  def call(self, x, training=None):\n",
        "    sublayer1 = self.multihead_attention((x, x, x), training=training)\n",
        "    sublayer1 = self.dropout1(sublayer1, training=training)\n",
        "    layernorm1 = self.layer_normalization1(x + sublayer1)\n",
        "    \n",
        "    sublayer2 = self.linear2(self.linear1(layernorm1))\n",
        "    sublayer1 = self.dropout2(sublayer2, training=training)\n",
        "    layernorm2 = self.layer_normalization2(layernorm1 + sublayer2)\n",
        "    return layernorm2\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return input_shape\n",
        "\n",
        "class TransformerDecoder(Layer):\n",
        "  def __init__(self, dropout=.0, attention_dropout=.0, **kwargs):\n",
        "    super(TransformerDecoder, self).__init__(**kwargs)\n",
        "    self.dropout_rate = dropout\n",
        "    self.attention_dropout_rate = attention_dropout\n",
        "  def build(self, input_shape):\n",
        "    self.multihead_self_attention = MultiHeadAttention_my(dropout=self.attention_dropout_rate, masked=True)\n",
        "    self.dropout1 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization1 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    \n",
        "    self.multihead_encoder_attention = MultiHeadAttention_my(dropout=self.attention_dropout_rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization2 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    \n",
        "    self.linear1 = Dense(input_shape[-1] * 4, input_shape=input_shape, activation='relu',\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.linear2 = Dense(input_shape[-1], input_shape=self.linear1.compute_output_shape(input_shape),\n",
        "                        kernel_initializer='glorot_uniform', bias_initializer='glorot_uniform')\n",
        "    self.dropout3 = tf.keras.layers.Dropout(self.dropout_rate)\n",
        "    self.layer_normalization3 = LayerNormalization(input_shape=input_shape, epsilon=1e-6)\n",
        "    super(TransformerDecoder, self).build(input_shape)\n",
        "  def call(self, x, hidden, training=None):\n",
        "    sublayer1 = self.multihead_self_attention((x, x, x))\n",
        "    sublayer1 = self.dropout1(sublayer1, training=training)\n",
        "    layernorm1 = self.layer_normalization1(x + sublayer1)\n",
        "    \n",
        "    sublayer2 = self.multihead_encoder_attention((x, hidden, hidden))\n",
        "    sublayer2 = self.dropout2(sublayer2, training=training)\n",
        "    layernorm2 = self.layer_normalization2(layernorm1 + sublayer2)\n",
        "    \n",
        "    sublayer3 = self.linear2(self.linear1(layernorm1))\n",
        "    sublayer3 = self.dropout3(sublayer3, training=training)\n",
        "    layernorm3 = self.layer_normalization2(layernorm2 + sublayer3)\n",
        "    return layernorm3\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return input_shape\n",
        "\n",
        "class SinusoidalPositionalEncoding(Layer): # This is a TensorFlow implementation of https://github.com/graykode/nlp-tutorial/blob/master/5-1.Transformer/Transformer_Torch.ipynb\n",
        "  def __init__(self):\n",
        "    super(SinusoidalPositionalEncoding, self).__init__()\n",
        "    self.sinusoidal_encoding = np.array([self.get_positional_angle(pos) for pos in range(max_token_length)], dtype=np.float32)\n",
        "    self.sinusoidal_encoding[:, 0::2] = np.sin(self.sinusoidal_encoding[:, 0::2])\n",
        "    self.sinusoidal_encoding[:, 1::2] = np.cos(self.sinusoidal_encoding[:, 1::2])\n",
        "    self.sinusoidal_encoding = tf.cast(self.sinusoidal_encoding, dtype=tf.float32) # Casting the array to Tensor for slicing\n",
        "  def call(self, x):\n",
        "    return x + self.sinusoidal_encoding[:tf.shape(x)[1]]\n",
        "  def compute_output_shape(self, input_shape):\n",
        "    return input_shape\n",
        "  def get_angle(self, pos, dim):\n",
        "    return pos / np.power(10000, 2 * (dim // 2) / d_model)\n",
        "  def get_positional_angle(self, pos):\n",
        "    return [self.get_angle(pos, dim) for dim in range(d_model)]\n",
        "\n",
        "class Transformer(Model):\n",
        "  def __init__(self, dropout=.1, attention_dropout=.0, **kwargs):\n",
        "    super(Transformer, self).__init__(**kwargs)\n",
        "    self.encoding_embedding = Embedding(VOCAB_SIZE, d_model)\n",
        "    self.decoding_embedding = Embedding(VOCAB_SIZE, d_model)\n",
        "    self.pos_encoding = SinusoidalPositionalEncoding()\n",
        "    self.encoder = [TransformerEncoder(dropout=dropout, attention_dropout=attention_dropout) for i in range(n_encoder_layers)]\n",
        "    self.decoder = [TransformerDecoder(dropout=dropout, attention_dropout=attention_dropout) for i in range(n_decoder_layers)]\n",
        "    self.decoder_final = Dense(VOCAB_SIZE, input_shape=(None, d_model))\n",
        "  def call(self, inputs, training=None): # Source_sentence and decoder_input\n",
        "    source_sentence, decoder_input = inputs\n",
        "    embedded_source = self.encoding_embedding(source_sentence)\n",
        "    encoder_output = self.pos_encoding(embedded_source)\n",
        "    for encoder_unit in self.encoder:\n",
        "      encoder_output = encoder_unit(encoder_output, training=training)\n",
        "    \n",
        "    embedded_target = self.decoding_embedding(decoder_input)\n",
        "    decoder_output = self.pos_encoding(embedded_target)\n",
        "    for decoder_unit in self.decoder:\n",
        "      decoder_output = decoder_unit(decoder_output, encoder_output, training=training)\n",
        "    if training:\n",
        "      decoder_output = self.decoder_final(decoder_output)\n",
        "      decoder_output = tf.nn.softmax(decoder_output, axis=-1)\n",
        "    else:\n",
        "      decoder_output = self.decoder_final(decoder_output[:, -1:, :])\n",
        "      decoder_output = tf.nn.softmax(decoder_output, axis=-1)\n",
        "    return decoder_output"
      ],
      "metadata": {
        "id": "KvsIOzPnzt4F"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transformer = Transformer(dropout=0.3) # Instantiating a new transformer model\n",
        "train = [tf.cast(train_src, dtype=tf.float32), tf.cast(train_tgt, dtype=tf.float32)] # Cast the tuples to tensors\n",
        "validation = [tf.cast(train_dev_src, dtype=tf.float32), tf.cast(train_dev_tgt, dtype=tf.float32)]"
      ],
      "metadata": {
        "id": "eRsGRYTR6XQW"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.load_weights(\"model_weights\") # Loading prev trained model weights"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WswPMfLjBc8P",
        "outputId": "6c3690ca-4dde-4b06-a74c-fd737897f64b"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f2a05be2f50>"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.build([train_src.shape, train_tgt.shape])\n",
        "transformer.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IHXUpv-0eh-b",
        "outputId": "a6a6fa19-0345-400b-f7b9-4f6a63f60f05"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"transformer\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       multiple                  8192000   \n",
            "                                                                 \n",
            " embedding_1 (Embedding)     multiple                  8192000   \n",
            "                                                                 \n",
            " sinusoidal_positional_encod  multiple                 0         \n",
            " ing (SinusoidalPositionalEn                                     \n",
            " coding)                                                         \n",
            "                                                                 \n",
            " transformer_encoder (Transf  multiple                 789760    \n",
            " ormerEncoder)                                                   \n",
            "                                                                 \n",
            " transformer_encoder_1 (Tran  multiple                 789760    \n",
            " sformerEncoder)                                                 \n",
            "                                                                 \n",
            " transformer_decoder (Transf  multiple                 1052928   \n",
            " ormerDecoder)                                                   \n",
            "                                                                 \n",
            " transformer_decoder_1 (Tran  multiple                 1052928   \n",
            " sformerDecoder)                                                 \n",
            "                                                                 \n",
            " dense (Dense)               multiple                  8224000   \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 28,293,376\n",
            "Trainable params: 28,293,376\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {
        "id": "0JkChx5Ndp1s"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])"
      ],
      "metadata": {
        "id": "InXkWf-LawJi"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCHS = 10\n",
        "checkpoint_filepath = 'checkpoints'\n",
        "model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_filepath,\n",
        "    save_weights_only=True,\n",
        "    monitor = \"accuracy\",\n",
        "    mode='max',\n",
        "    save_best_only=True)"
      ],
      "metadata": {
        "id": "LMttctIwaNxG"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with tf.device('/device:GPU:0'):\n",
        "    history = transformer.fit(train, train_labels.reshape(train_labels.shape[0], train_labels.shape[1],1), batch_size=256, epochs=EPOCHS, callbacks=[model_checkpoint_callback])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bzdsT8kmBEMo",
        "outputId": "ceda01a6-c789-4ad4-aec8-b9f589680663"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "605/605 [==============================] - 397s 656ms/step - loss: 0.9338 - accuracy: 0.8415\n",
            "Epoch 2/10\n",
            "605/605 [==============================] - 409s 676ms/step - loss: 0.9032 - accuracy: 0.8442\n",
            "Epoch 3/10\n",
            "605/605 [==============================] - 397s 656ms/step - loss: 0.8752 - accuracy: 0.8466\n",
            "Epoch 4/10\n",
            "605/605 [==============================] - 396s 655ms/step - loss: 0.8483 - accuracy: 0.8491\n",
            "Epoch 5/10\n",
            "605/605 [==============================] - 396s 655ms/step - loss: 0.8232 - accuracy: 0.8516\n",
            "Epoch 6/10\n",
            "605/605 [==============================] - 396s 655ms/step - loss: 0.7996 - accuracy: 0.8539\n",
            "Epoch 7/10\n",
            "605/605 [==============================] - 396s 655ms/step - loss: 0.7769 - accuracy: 0.8564\n",
            "Epoch 8/10\n",
            "605/605 [==============================] - 396s 654ms/step - loss: 0.7556 - accuracy: 0.8586\n",
            "Epoch 9/10\n",
            "605/605 [==============================] - 396s 655ms/step - loss: 0.7351 - accuracy: 0.8610\n",
            "Epoch 10/10\n",
            "605/605 [==============================] - 396s 655ms/step - loss: 0.7154 - accuracy: 0.8633\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "transformer.save_weights(\"model_weights-2l-8h-loss-changed\")\n",
        "# transformer.save(\"./Models/\")"
      ],
      "metadata": {
        "id": "6tmeafiFdZ_U"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 10))\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train'], loc='upper left')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "JbSCoj0cft6O",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 621
        },
        "outputId": "1c76e6d0-f083-4992-a3d0-82e5934a6e40"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm0AAAJcCAYAAACrLZO2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3jV9f3+8ftFCIS99xBk7xWSULVuq+LWInsP615tbWtbq/122Na2WheBsKfbOiuttA6y2AiyR8IMIyEJZJ73748c+4sUJUBOPmc8H9eVqznn5Jzcx6o8/ZxlzjkBAAAguFXzegAAAADOjGgDAAAIAUQbAABACCDaAAAAQgDRBgAAEAKINgAAgBBAtAEIW2Y228x+XcGf3WVmVwV6EwCcK6INAAAgBBBtABDkzKy61xsAeI9oA+Ap/8OSPzSzdWaWb2YzzayFmb1vZrlmtszMGpX7+ZvM7Aszyzaz5WbWo9xlA8xslf96SyTFnPK7bjCzNf7rfm5mfSu4caiZrTaz42aWYWZPnHL5xf7by/ZfPt5/fi0z+5OZ7TazHDP71H/eZWaWeZq/Dlf5v3/CzF41s/lmdlzSeDOLM7MV/t+x38z+ZmY1yl2/l5l9ZGZHzeygmf3UzFqa2Qkza1Lu5waaWZaZRVfkvgMIHkQbgGBwu6SrJXWVdKOk9yX9VFIzlf176n5JMrOukhZJetB/2XuS/m5mNfwB86akeZIaS3rFf7vyX3eApCRJ0yQ1kfSypLfNrGYF9uVLGiupoaShkn5gZrf4b/cC/97n/Jv6S1rjv94fJQ2S9B3/ph9J8lXwr8nNkl71/84FkkolPSSpqaQhkq6UdLd/Qz1JyyR9IKm1pM6S/umcOyBpuaRh5W53jKTFzrniCu4AECSINgDB4Dnn3EHn3F5Jn0hKcc6tds4VSHpD0gD/z90p6V3n3Ef+6PijpFoqi6IESdGS/uKcK3bOvSoprdzvmCrpZedcinOu1Dk3R1Kh/3rfyjm33Dm33jnnc86tU1k4Xuq/eKSkZc65Rf7fe8Q5t8bMqkmaKOkB59xe/+/83DlXWMG/Jiucc2/6f+dJ59xK51yyc67EObdLZdH51YYbJB1wzv3JOVfgnMt1zqX4L5sjabQkmVmUpBEqC1sAIYZoAxAMDpb7/uRpTtf1f99a0u6vLnDO+SRlSGrjv2yvc86Vu+7uct9fIOkR/8OL2WaWLamd/3rfyszizexj/8OKOZLuUtkRL/lvY/tprtZUZQ/Pnu6yisg4ZUNXM3vHzA74HzL9TQU2SNJbknqaWUeVHc3Mcc6lnuMmAB4i2gCEkn0qiy9JkpmZyoJlr6T9ktr4z/tK+3LfZ0j6P+dcw3JftZ1ziyrwexdKeltSO+dcA0kvSfrq92RI6nSa6xyWVPANl+VLql3ufkSp7KHV8twpp1+U9KWkLs65+ip7+Lj8hgtPN9x/tHKpyo62jRFH2YCQRbQBCCVLJQ01syv9T6R/RGUPcX4uaYWkEkn3m1m0md0mKa7cdRMl3eU/amZmVsf/AoN6Ffi99SQddc4VmFmcyh4S/coCSVeZ2TAzq25mTcysv/8oYJKkZ8ystZlFmdkQ/3PotkiK8f/+aEmPSzrTc+vqSTouKc/Mukv6QbnL3pHUysweNLOaZlbPzOLLXT5X0nhJN4loA0IW0QYgZDjnNqvsiNFzKjuSdaOkG51zRc65Ikm3qSxOjqrs+W+vl7tuuqQpkv4m6Zikbf6frYi7JT1pZrmSfqGyePzqdvdIul5lAXlUZS9C6Oe/+FFJ61X23Lqjkn4vqZpzLsd/mzNUdpQwX9LXXk16Go+qLBZzVRagS8ptyFXZQ583Sjogaauky8td/pnKXgCxyjlX/iFjACHEvv70DwBAODKzf0la6Jyb4fUWAOeGaAOAMGdmgyV9pLLn5OV6vQfAueHhUQAIY2Y2R2Xv4fYgwQaENo60AQAAhACOtAEAAISAiPgQ4qZNm7oOHTp4PQMAAOCMVq5cedg5d+p7N0ZGtHXo0EHp6elezwAAADgjMzvtW/Pw8CgAAEAIINoAAABCANEGAAAQAiLiOW2nU1xcrMzMTBUUFHg9JaBiYmLUtm1bRUdHez0FAACch4iNtszMTNWrV08dOnSQmXk9JyCcczpy5IgyMzPVsWNHr+cAAIDzELEPjxYUFKhJkyZhG2ySZGZq0qRJ2B9NBAAgEkRstEkK62D7SiTcRwAAIkFERxsAAECoINo8kp2drRdeeOGsr3f99dcrOzs7AIsAAEAwI9o88k3RVlJS8q3Xe++999SwYcNAzQIAAEEqYl896rXHHntM27dvV//+/RUdHa2YmBg1atRIX375pbZs2aJbbrlFGRkZKigo0AMPPKCpU6dK+v8fyZWXl6frrrtOF198sT7//HO1adNGb731lmrVquXxPQMAAIFAtEn61d+/0MZ9xyv1Nnu2rq9f3tjrGy//3e9+pw0bNmjNmjVavny5hg4dqg0bNvz3rTmSkpLUuHFjnTx5UoMHD9btt9+uJk2afO02tm7dqkWLFikxMVHDhg3Ta6+9ptGjR1fq/QAAAMGBaAsScXFxX3svtWeffVZvvPGGJCkjI0Nbt279n2jr2LGj+vfvL0kaNGiQdu3aVWV7AQBA1SLapG89IlZV6tSp89/vly9frmXLlmnFihWqXbu2LrvsstO+11rNmjX/+31UVJROnjxZJVsBAEDV44UIHqlXr55yc3NPe1lOTo4aNWqk2rVr68svv1RycnIVrwMAAMGGI20eadKkiS666CL17t1btWrVUosWLf572bXXXquXXnpJPXr0ULdu3ZSQkODhUgAAEAzMOef1hoCLjY116enpXztv06ZN6tGjh0eLqlYk3VcAAEKdma10zsWeej4PjwIAAIQAog0AACAERHS0RcJDw5FwHwEAiAQRG20xMTE6cuRIWEeNc05HjhxRTEyM11MAAMB5ithXj7Zt21aZmZnKysryekpAxcTEqG3btl7PAAAg5BWV+FSjunfHuyI22qKjo7/2CQQAAADf5M3Ve/WXZVu0ZNoQtajvzSNYEfvwKAAAQEUkfbpTDy5Zo5YNYlS7RpRnOyL2SBsAAMC3cc7pj//YrOc/3q7v9Wqhvw4foJhoog0AACBolPqcHn9zvRalZmhEXDv9+pY+iqpmnm4i2gAAAMopKC7Vg4vX6IMvDujeyzvrkWu6yszbYJOINgAAgP/KLSjWlLnpSt5xVL+4oacmXhw8L1ok2gAAACRl5RZq/KxUbT6Qq7/c2V+3DGjj9aSvIdoAAEDEyzh6QmNmpujA8QIljovV5d2aez3pfxBtAAAgom3af1xjk1JVVOLTgskJGnRBI68nnRbRBgAAIlbqzqOaNCdNdWpU1yt3DVHXFvW8nvSNiDYAABCRlm08qHsWrlKbhrU0d1Kc2jaq7fWkb0W0AQCAiPNKeoYee329erWur1njB6tJ3ZpeTzojog0AAESU6f/Zrt+896Uu7txUL40ZpLo1QyOHQmMlAADAeXLO6Xfvf6mX/7NDQ/u20jPD+qlmde8+lupsEW0AACDslZT69Njr6/XqykyNSbhAT9zUy/OPpTpbRBsAAAhrBcWlunfhKi3bdEgPXtVFD1zZJSg+lupsEW0AACBs5Zws1pQ56UrbfVRP3txLY4d08HrSOSPaAABAWDp0vEBjk1K1PStPzw4foBv7tfZ60nkh2gAAQNjZdThfY5JSdCSvSEnjB+uSLs28nnTeiDYAABBWNuzN0fhZqSr1OS2akqB+7Rp6PalSEG0AACBsrNh+RFPnpqteTHXNnRqvzs3rej2p0hBtAAAgLHyw4YDuX7xa7RvX1rxJcWrVoJbXkyoV0QYAAELe4tQ9+ukb69WvXUMljRusRnVqeD2p0hFtAAAgZDnn9MLy7frDh5t1addmenH0QNWuEZ55E573CgAAhD2fz+nX725S0mc7dUv/1vrD9/spOqqa17MChmgDAAAhp7jUpx+9uk5vrN6rCRd10M+H9lS1EPtYqrNFtAEAgJByoqhEdy9YpeWbs/TD73XT3Zd1CsmPpTpbRBsAAAgZ2SeKNHF2mtZkZOu3t/XRiLj2Xk+qMkQbAAAICQdyCjQ2KUW7Dp/QC6MG6trerbyeVKWINgAAEPS2Z+Vp7MxU5Zws1uyJg/WdTk29nlTliDYAABDU1mVma/ysNJmkxVMT1LtNA68neYJoAwAAQevTrYc1bV66GtWpoXmT4tWxaR2vJ3mGaAMAAEHp3XX79eCS1erUrK7mTIxTi/oxXk/yFNEGAACCzrzk3frFWxsUe0EjzRg3WA1qRXs9yXNEGwAACBrOOT37z23687ItuqpHc/1t5EDFREd5PSsoEG0AACAo+HxOT/z9C81dsVu3D2yr39/eR9XD+GOpzhbRBgAAPFdU4tPDS9fonXX7NfW7F+on13WPiE85OBtEGwAA8FR+YYnumr9Sn2w9rJ9c113TLu3k9aSgRLQBAADPHM0v0oTZadqwN0dP39FXw2LbeT0paBFtAADAE3uzT2rMzBTtPXZSL40epKt7tvB6UlAj2gAAQJXbejBXY5NSlVdYonmT4hXXsbHXk4Ie0QYAAKrUqj3HNHF2mqKjqmnJ1CHq2bq+15NCAtEGAACqzL+3ZOmueSvVvH5NzZsYr/ZNans9KWQQbQAAoEq8tWavHlm6Vl1b1NOciXFqVq+m15NCCtEGAAACbtZnO/Wrv29UfMfGShwXq/oxfCzV2SLaAABAwDjn9MxHW/Tcv7bpmp4t9OyIAXws1Tki2gAAQECU+px+/tYGLUzZo+GD2+nXt/TmY6nOA9EGAAAqXWFJqR5cvEbvbzigey7vpEev6cbHUp0nog0AAFSq3IJiTZu3Up9vP6Kf39BTky7u6PWksEC0AQCASnM4r1DjZ6Xqy/25+vOd/XTrgLZeTwobRBsAAKgUGUdPaMzMFB04XqDEsbG6vHtzryeFFaINAACct80HcjVmZooKS3xaMDlBgy5o5PWksEO0AQCA87I2I1tjk1JVKzpKr9w1RF1b1PN6Ulgi2gAAwDlL23VUE2alqVGdaC2cnKB2jflYqkAh2gAAwDn5dOthTZmbrlYNY7RwcoJaNojxelJY4x3uAADAWfvXlwc1cU6aLmhSW0umDiHYqgBH2gAAwFl5b/1+3b9otXq2rq+5E+PUsHYNrydFBI60AQCACnt9VabuXbhK/ds11PzJ8QRbFeJIGwAAqJCFKXv0szfX6zudmihxbKxq1yAjqhJ/tQEAwBnN/HSnnnpno67o3lwvjBqomOgorydFHKINAAB8q7/9a6v++I8tur5PS/3lzgGqUZ1nV3mBaAMAAKflnNMf/7FZz3+8XbcNaKOn7+ir6lEEm1eINgAA8D+cc3rynY2a9dkujYhrr/+7pbeqVTOvZ0U0og0AAHyNz+f0szc3aFHqHk24qIN+cUNPmRFsXiPaAADAf5WU+vTDV9fpjdV7dc/lnfToNd0ItiBBtAEAAElSUYlPDyxerfc3HNCj13TVvVd08XoSyiHaAACACopLdfeCVfrXl4f08xt6atLFHb2ehFMQbQAARLgTRSWaMjddn28/ov+7tbdGxV/g9SScBtEGAEAEyy0o1sTZaVq5+5j+9P1+um1gW68n4RsQbQAARKjsE0Uam5SqjfuO67kRAzW0byuvJ+FbEG0AAESgw3mFGj0jRTsO5+vlMYN0ZY8WXk/CGRBtAABEmAM5BRo5I1n7swuUNG6wLu7S1OtJqICAfhaFmV1rZpvNbJuZPXaay9ub2cdmttrM1pnZ9eUu62tmK8zsCzNbb2Yx/vMH+U9vM7NnjTePAQCgwjKOntCwl1fo0PFCzZ0UR7CFkIBFm5lFSXpe0nWSekoaYWY9T/mxxyUtdc4NkDRc0gv+61aXNF/SXc65XpIuk1Tsv86LkqZI6uL/ujZQ9wEAgHCyIytPw15eoZyTxVowOV6DOzT2ehLOQiCPtMVJ2uac2+GcK5K0WNLNp/yMk1Tf/30DSfv8318jaZ1zbq0kOeeOOOdKzayVpPrOuWTnnJM0V9ItAbwPAACEhc0HcjXs5WQVlfi0eGqC+rVr6PUknKVARlsbSRnlTmf6zyvvCUmjzSxT0nuS7vOf31WSM7MPzWyVmf2o3G1mnuE2JUlmNtXM0s0sPSsr6/zuCQAAIWzD3hwNn75CUdWkJdOGqEer+me+EoJOQJ/TVgEjJM12zrWVdL2keWZWTWUvkLhY0ij//95qZleezQ0756Y752Kdc7HNmjWr7N0AAISElbuPaURismrXqK6l04aoc/O6Xk/COQpktO2V1K7c6bb+88qbJGmpJDnnVkiKkdRUZUfQ/uOcO+ycO6Gyo3AD/dcv/65/p7tNAAAg6fPthzVmZoqa1q2pV+4aogua1PF6Es5DIKMtTVIXM+toZjVU9kKDt0/5mT2SrpQkM+uhsmjLkvShpD5mVtv/ooRLJW10zu2XdNzMEvyvGh0r6a0A3gcAAELS8s2HNGFWmto2qqUl0xLUumEtryfhPAXsfdqccyVmdq/KAixKUpJz7gsze1JSunPubUmPSEo0s4dU9qKE8f4XGBwzs2dUFn5O0nvOuXf9N323pNmSakl63/8FAAD8PthwQPctWqWuLepp3qR4Na5Tw+tJqARW1kjhLTY21qWnp3s9AwCAgHtrzV49vHSt+rZtoNkT4tSgVrTXk3CWzGylcy721PP5RAQAAMLEkrQ9euz19Yrv2Fgzxg1W3Zr8MR9O+H8TAIAwMPuznXri7xv13a7N9PLoQapVI8rrSahkRBsAACHuxeXb9fsPvtQ1PVvouZEDVLM6wRaOiDYAAEKUc05/XrZVz/5zq27s11rPDOun6Civ34IVgUK0AQAQgpxz+s17m5T4yU4Ni22r397WV1HVzOtZCCCiDQCAEOPzOf3i7Q2an7xH44ZcoF/e2EvVCLawR7QBABBCSn1OP35tnV5dmalpl16ox67trrL3m0e4I9oAAAgRxaU+PbRkjd5Zt18PXdVV91/ZmWCLIEQbAAAhoLCkVPcuXK2PNh7UT6/vrqnf7eT1JFQxog0AgCB3sqhUU+el65Oth/XUzb00ZkgHryfBA0QbAABBLK+wRBNnpyl911E9fUdfDYtt5/UkeIRoAwAgSOWcKNa4WalavzdHfxk+QDf1a+31JHiIaAMAIAgdySvUmJmp2nYoTy+OGqhrerX0ehI8RrQBABBkDh0v0KgZKdpz9IQSx8Xq0q7NvJ6EIEC0AQAQRPZmn9SoxGQdyi3U7AlxGtKpideTECSINgAAgsSuw/kaNSNFxwuKNX9yvAa2b+T1JAQRog0AgCCw9WCuRs1IUXGpT4umJKh3mwZeT0KQIdoAAPDYF/tyNGZmqqKqmZZMG6KuLep5PQlBqJrXAwAAiGSr9xzTiOnJiqleTUsJNnwLjrQBAOCRlB1HNHF2mprUramFU+LVtlFtrychiBFtAAB44D9bsjR1XrraNKylBZMT1LJBjNeTEOSINgAAqthHGw/qngWr1Kl5Xc2bFKemdWt6PQkhgGgDAKAK/X3tPj20ZI16ta6vORPj1LB2Da8nIUQQbQAAVJFXV2bqR6+uVewFjTVzfKzqxUR7PQkhhGgDAKAKzEverZ+/uUGXdGmql8cMUu0a/BGMs8PfMQAABJBzTn/71zb96aMtuqpHc/1t5EDFREd5PQshiGgDACBASn1Ov3hrgxak7NFtA9rod7f3VY3qvEUqzg3RBgBAABQUl+qBxav14RcHddelnfTja7vJzLyehRBGtAEAUMmyTxRp8px0rdxzTL+8sacmXNTR60kIA0QbAACVaF/2SY1LStXuIyf03IgBuqFva68nIUwQbQAAVJLNB3I1LilV+YUlmjMxTkM6NfF6EsII0QYAQCVI3nFEU+amq1Z0lJbeNUQ9WtX3ehLCDNEGAMB5en/9fj2wZI3aNaqlORPj+OB3BATRBgDAeZjz+S498fcvNKBdQ80cN1iN6vCxVAgMog0AgHPgnNMfPtysF5Zv11U9Wui5EQNUqwZvmovAIdoAADhLxaU+Pfbaer22KlMj4trrqZt7qXoUb5qLwCLaAAA4C/mFJbpn4Sot35ylh67qqvuv7Myb5qJKEG0AAFTQ4bxCTZydpg17c/Tb2/poRFx7rychghBtAABUwJ4jJzQ2KUUHjhdo+phYXdWzhdeTEGGINgAAzmB9Zo4mzE5Vic9pweQEDbqgkdeTEIGINgAAvsV/tmTpB/NXqmHtGlo8MU6dm9f1ehIiFNEGAMA3eGN1pn74yjp1bl5XcybGqUX9GK8nIYIRbQAAnMI5p+n/2aHfvv+lhlzYRC+PHaT6MdFez0KEI9oAACjH53N66t2NmvXZLt3Qt5X+NKyfalbnTXPhPaINAAC/wpJSPbx0rd5dt18TL+qox4f2ULVqvAcbggPRBgCApOMFxZo2d6VW7Diin17fXVMuuZA3zUVQIdoAABHv4PECjUtK1bZDefrLnf11y4A2Xk8C/gfRBgCIaNsO5WpcUpqyTxQpafxgfbdrM68nAadFtAEAItbK3cc0aU6aqlczLZk2RL3bNPB6EvCNiDYAQET6aONB3bdolVrWj9HcifFq36S215OAb0W0AQAizqLUPfrZG+vVp00DzRw/WE3r1vR6EnBGRBsAIGI45/TXf27VX5Zt1WXdmun5kQNVpyZ/FCI08HcqACAilJT69PO3NmhRaobuGNRWv72tj6Kjqnk9C6gwog0AEPZOFpXqvkWrtWzTQd1zeSc9ek033oMNIYdoAwCEtWP5RZo0J02rM7L15M29NHZIB68nAeeEaAMAhK3MYyc0LilVGcdO6oWRA3Vdn1ZeTwLOGdEGAAhLG/cd1/hZqTpZXKp5E+MUf2ETrycB54VoAwCEnc+3H9a0uStVp2Z1vXrXd9StZT2vJwHnjWgDAISVd9bt08NL1uqCJrU1Z2KcWjes5fUkoFIQbQCAsJH06U499e5GxV7QSIljY9Wwdg2vJwGVhmgDAIQ8n8/p9x9+qZf/vUPf69VCfx0+QDHRUV7PAioV0QYACGlFJT79+LV1emP1Xo1OaK9f3dRbUdV4DzaEH6INABCy8gpL9IP5K/XJ1sN69JquuufyzrxpLsIW0QYACElZuYWaMDtVm/bn6unb+2rY4HZeTwICimgDAIScXYfzNTYpVVm5hUocO0hXdG/h9SQg4Ig2AEBIWZuRrYmz0+RzTgunxGtA+0ZeTwKqBNEGAAgZyzcf0g/mr1KTujU0d2KcLmxW1+tJQJUh2gAAIeHVlZl67LV16taynmZNGKzm9WK8ngRUKaINABDUnHN6Yfl2/eHDzbqocxO9NHqQ6sVEez0LqHJEGwAgaJX6nJ78+xeas2K3bu7fWn+4o59qVK/m9SzAE0QbACAoFRSX6uGla/Te+gOacklH/eS6HqrGm+YighFtAICgk3OyWFPmpit151E9PrSHJl9yodeTAM8RbQCAoLI/56TGJ6Vpx+E8/XV4f93cv43Xk4CgQLQBAILG1oO5GpeUquMFJZo9IU4XdW7q9SQgaBBtAICgkLbrqCbPSVeN6tW0ZFqCerVu4PUkIKgQbQAAzy3beFD3LFylNg1rac7EOLVrXNvrSUDQIdoAAJ76YMMB3btwlXq1rq9ZE+LUuE4NrycBQYloAwB45t11+3X/4tXq27aB5kyMU33eNBf4RrxDIQDAE2+v3af7F6/WgHYNNZdgA86II20AgCr35uq9enjpGsV2aKxZ4werTk3+OALOhCNtAIAq9erKTD20dI3iOzbR7AkEG1BR/JMCAKgyS9My9OPX1+miTk2VODZWtWpEeT0JCBkcaQMAVImFKXv0o9fW6eLOTTVjHMEGnC2iDQAQcPOSd+unb6zXZd2aKXFsrGKiCTbgbPHwKAAgoGZ/tlNP/H2jruzeXC+MHqia1Qk24FwQbQCAgJnxyQ79+t1NurpnCz0/cqBqVOcBHuBcEW0AgICY/p/t+s17X+q63i317IgBio4i2IDzQbQBACrdC8u36ekPNmto31b6y539CTagEhBtAIBK9dw/t+pPH23RTf1a65lh/VSdYAMqBdEGAKgUzjn9ZdlW/fWfW3XbgDb6w/f7KaqaeT0LCBtEGwDgvDnn9MxHW/Tcv7bpjkFt9fvb+xJsQCUj2gAA58U5p6c/3KwXl2/X8MHt9Jtb+6gawQZUOqINAHDOnHP67ftfavp/dmhkfHv9+ubeBBsQIEQbAOCcOOf01DublPTZTo0dcoF+dVMvmRFsQKAQbQCAs+ac0xNvf6E5K3ZrwkUd9IsbehJsQIARbQCAs+LzOf3i7Q2an7xHUy7pqJ9e34NgA6oA0QYAqDCfz+lnb67XotQM3XVpJ/342m4EG1BFiDYAQIWU+pwee22dXlmZqXsv76xHrulKsAFViGgDAJxRqc/ph6+u1eur9uqBK7vowau6EGxAFSPaAADfqqTUp0deWau31uzTw1d31f1XdvF6EhCRiDYAwDcqKfXpwSVr9M66/frh97rpnss7ez0JiFhEGwDgtIpLfXpg8Wq9t/6AfnJdd027tJPXk4CIRrQBAP5HUYlP9y1apQ+/OKjHh/bQ5Esu9HoSEPGINgDA1xSWlOqeBau0bNMhPXFjT42/qKPXkwCIaAMAlFNQXKofzF+pjzdn6ambe2nMkA5eTwLgVy2QN25m15rZZjPbZmaPneby9mb2sZmtNrN1Zna9//wOZnbSzNb4v14qd53l/tv86rLmgbwPABApCopLNW1eWbD95tY+BBsQZAJ2pM3MoiQ9L+lqSZmS0szsbefcxnI/9rikpc65F82sp6T3JHXwX7bdOdf/G25+lHMuPUDTASDinCwq1dR56fp022E9fXtfDRvczutJAE4RyCNtcZK2Oed2OOeKJC2WdPMpP+Mk1fd/30DSvgDuAQCcxomiEk2ak6ZPtx3WH+7oR7ABQSqQ0dZGUka505n+88p7QtJoM8tU2VG2+8pd1tH/sOm/zeySU643y//Q6M/tG96S28ymmlm6maVnZWWd3z0BgDCVX1iiCbPSlLzjiJ4Z1k93DGrr9SQA3yCgz2mrgBGSZjvn2kq6XtI8M6smab+k9hORxTsAACAASURBVM65AZIelrTQzL46IjfKOddH0iX+rzGnu2Hn3HTnXKxzLrZZs2YBvyMAEGryCks0flaq0ncf01+GD9CtAwg2IJgFMtr2Sip/jL2t/7zyJklaKknOuRWSYiQ1dc4VOueO+M9fKWm7pK7+03v9/5sraaHKHoYFAJyF3IJijZ2ZolV7svXs8AG6qV9rrycBOINARluapC5m1tHMakgaLuntU35mj6QrJcnMeqgs2rLMrJn/hQwyswsldZG0w8yqm1lT//nRkm6QtCGA9wEAwk7OyWKNmZmqdZk5en7kAA3t28rrSQAqIGCvHnXOlZjZvZI+lBQlKck594WZPSkp3Tn3tqRHJCWa2UMqe1HCeOecM7PvSnrSzIol+STd5Zw7amZ1JH3oD7YoScskJQbqPgBAuMk5UawxSSnatP+4Xhg1UNf0aun1JAAVZM45rzcEXGxsrEtP5x1CAES2Y/lFGj0zRVsP5unF0QN1ZY8WXk8CcBpmttI5F3vq+XwiAgBEgKP5RRo1I0Xbs/L08thBurwb70sOhBqiDQDC3OG8Qo2ekaKdh/M1Y2ysvtuVV9QDoYhoA4AwlpVbqJGJyco4dkJJ4wfros5NvZ4E4BwRbQAQpg4dL9CIxGTtyy7QrPFxGtKpideTAJwHog0AwtCBnAKNTEzWgeMFmjMxTnEdG3s9CcB5ItoAIMzsyz6pEYnJOpJXpLkT4xTbgWADwgHRBgBhJPPYCY1ITFZ2frHmTorTwPaNvJ4EoJIQbQAQJjKOntDw6cnKLSjW/Mnx6teuodeTAFQiog0AwsDuI/kamZiivMISLZicoD5tG3g9CUAlI9oAIMTtPJyvkYnJKigu1cIp8erVmmADwhHRBgAhbHtWnkYmJqu41GnhlAT1aFXf60kAAoRoA4AQte1QrkYkpsg5p0VTEtStZT2vJwEIIKINAELQloO5GpmYLDPT4qkJ6tycYAPCXTWvBwAAzs6m/cc1fHqyqhFsQEThSBsAhJAv9uVo9IwU1awepUVTE9SxaR2vJwGoIhxpA4AQsT4zRyMTU1QrOkpLphFsQKThSBsAhIC1GdkaPTNF9WOitXhqgto1ru31JABVjCNtABDkUnYc0agZKWpYO1pLphFsQKQi2gAgiH28+ZDGJqWqRf2aWjptiNo2ItiASMXDowAQpN5Zt08PLl6j7q3qac6EODWpW9PrSQA8RLQBQBBanLpHP3ljvWIvaKSZ4werfky015MAeIxoA4AgM+OTHfr1u5t0addmemn0INWqEeX1JABBgGgDgCDhnNOfP9qiZ/+1TUP7tNKf7+yvGtV56jGAMkQbAAQBn8/pyXc2avbnu3RnbDv95rY+iqpmXs8CEESINgDwWEmpTz9+bb1eW5WpSRd31ONDe8iMYAPwdUQbAHiosKRU9y9arQ+/OKiHr+6q+67oTLABOC2iDQA8cqKoRNPmrdQnWw/rFzf01MSLO3o9CUAQI9oAwAM5J4s1cXaaVu85pj/c0Vffj23n9SQAQY5oA4AqlpVbqLFJqdp2KFcvjBqoa3u38noSgBBAtAFAFdqbfVKjZ6ToQE6BZo4brO92beb1JAAhgmgDgCqyIytPo2ekKLewRPMmxSm2Q2OvJwEIIUQbAFSBL/blaFxSqpyTFk9NUK/WDbyeBCDEEG0AEGArdx/V+FlpqlezuuZNjlenZnW9ngQgBBFtABBAn2zN0tS5K9WyQYzmT45Xm4a1vJ4EIERV6EPtzOx1MxtqZnwIHgBU0AcbDmjS7HRd0KS2lk4bQrABOC8VjbAXJI2UtNXMfmdm3QK4CQBC3qsrM3X3gpXq3aa+lkwdomb1ano9CUCIq1C0OeeWOedGSRooaZekZWb2uZlNMLPoQA4EgFAz+7OdevSVtfpOp6aaNyleDWrzr0kA56/CD3eaWRNJ4yVNlrRa0l9VFnEfBWQZAIQY55ye++dWPfH3jbqmZwvNGBerOjV56jCAylGhf5uY2RuSukmaJ+lG59x+/0VLzCw9UOMAIFQ45/Sb9zYp8ZOdum1gGz19e19Vj+JpwAAqT0X/E/BZ59zHp7vAORdbiXsAIOSU+px+9sZ6LU7L0LghF+iXN/ZStWrm9SwAYaai/xnY08wafnXCzBqZ2d0B2gQAIaOoxKf7F6/W4rQM3XdFZz1xE8EGIDAqGm1TnHPZX51wzh2TNCUwkwAgNJwsKtXUeel6d91+/fT67nrkmm4yI9gABEZFHx6NMjNzzjlJMrMoSTUCNwsAgltuQbEmzU5X2u6j+u1tfTQirr3XkwCEuYpG2wcqe9HBy/7T0/znAUDEOZpfpHFJqdq0/7ieHT5AN/Zr7fUkABGgotH2Y5WF2g/8pz+SNCMgiwAgiB3IKdDomSnKOHpCiWNjdXn35l5PAhAhKhRtzjmfpBf9XwAQkXYdztfomSnKPlGsuRPjFH9hE68nAYggFX2fti6Sfiupp6SYr853zl0YoF0AEFQ2H8jV6JkpKin1aeGUePVt2/DMVwKASlTRV4/OUtlRthJJl0uaK2l+oEYBQDBZk5GtYS+vUDWTlk4bQrAB8ERFo62Wc+6fksw5t9s594SkoYGbBQDB4fPthzUqMVkNakXr1bu+oy4t6nk9CUCEqugLEQrNrJqkrWZ2r6S9kuoGbhYAeG/ZxoO6e+EqdWhSW/MmxatF/ZgzXwkAAqSiR9oekFRb0v2SBkkaLWlcoEYBgNfeWrNX0+avVI+W9bRk6hCCDYDnznikzf9Gunc65x6VlCdpQsBXAYCH5iXv1i/e2qD4jo01Y9xg1a1Z0QclACBwzvhvIudcqZldXBVjAMBrLyzfpqc/2KwruzfX86MGKiY6yutJACCp4s9pW21mb0t6RVL+V2c6514PyCoAqGLOOT394Wa9uHy7burXWn8a1k/RURV9BgkABF5Foy1G0hFJV5Q7z0ki2gCEPJ/P6edvbdCClD0aFd9eT97cW1HV+OB3AMGlop+IwPPYAISl4lKfHn1lrd5as0/TLr1Qj13bXWYEG4DgU9FPRJilsiNrX+Ocm1jpiwCgihQUl+rehau1bNNB/ejabrr7ss5eTwKAb1TRh0ffKfd9jKRbJe2r/DkAUDXyCks0ZU66Vuw4oqdu7qUxQzp4PQkAvlVFHx59rfxpM1sk6dOALAKAAMs+UaRxs9K0YW+O/nxnP906oK3XkwDgjM71zYe6SGpemUMAoCocOl6gMTNTtfNwvl4cNVDX9Grp9SQAqJCKPqctV19/TtsBST8OyCIACJCMoyc0emaKsnILNWvCYF3UuanXkwCgwir68CifkAwgpG07lKvRM1J1srhUCybHa0D7Rl5PAoCzUqF3jjSzW82sQbnTDc3slsDNAoDKsz4zR99/aYVKfE5LpiUQbABCUkXf7vuXzrmcr04457Il/TIwkwCg8qTuPKqRicmqXaO6Xr1riLq3rO/1JAA4JxV9IcLp4o5PUAYQ1D7efEh3zVupto1qaf7keLVqUMvrSQBwzip6pC3dzJ4xs07+r2ckrQzkMAA4H++s26cpc9LVuXldLZ02hGADEPIqGm33SSqStETSYkkFku4J1CgAOB9L0vbo/kWrNaB9Qy2amqAmdWt6PQkAzltFXz2aL+mxAG8BgPM245Md+vW7m3Rp12Z6afQg1aoR5fUkAKgUFX316Edm1rDc6UZm9mHgZgHA2XHO6Zl/bNav392koX1aKXFsLMEGIKxU9MUETf2vGJUkOeeOmRmfiAAgKPh8Tk++s1GzP9+lYbFt9dvb+iqqmnk9CwAqVUWf0+Yzs/ZfnTCzDvr6JyQAgCd8PqefvblBsz/fpUkXd9TvbyfYAISnih5p+5mkT83s35JM0iWSpgZsFQBUgM/n9NM31mtxWobuubyTHr2mm8wINgDhqaIvRPjAzGJVFmqrJb0p6WQghwHAt/H5nB57fZ2Wpmfqvis66+GruxJsAMJaRT8wfrKkByS1lbRGUoKkFZKuCNw0ADi9Up/Tj19bp1dXZur+K7vooau6EGwAwl5Fn9P2gKTBknY75y6XNEBS9rdfBQAqX6nP6YevrtWrKzP14FVdOMIGIGJUNNoKnHMFkmRmNZ1zX0rqFrhZAPC/Sn1Oj76yVq+v2quHr+6qB6/q6vUkAKgyFX0hQqb/fdrelPSRmR2TtDtwswDg60pKfXrklbV6a80+PXpNV917RRevJwFAlaroCxFu9X/7hJl9LKmBpA8CtgoAyikp9enhpWv19tp9+uH3uumeyzt7PQkAqlxFj7T9l3Pu34EYAgCnU1Lq04NL1uiddfv142u76weXdfJ6EgB44qyjDQCqSnGpTw8uXqN31+/XT6/vrqnfJdgARC6iDUBQKi716f5Fq/X+hgN6fGgPTb7kQq8nAYCniDYAQaeoxKf7Fq3Sh18c1M9v6KlJF3f0ehIAeI5oAxBUikp8umfhKn208aB+eWNPTbiIYAMAiWgDEEQKS0p1z4JVWrbpkH51Uy+N+04HrycBQNAg2gAEhcKSUt09f5X++eUhPXVzL40Z0sHrSQAQVIg2AJ4rKC7VD+av1Mebs/TrW3prdMIFXk8CgKBDtAHwVEFxqe6av1LLN2fpN7f20cj49l5PAoCgRLQB8ExBcammzlup/2zJ0u9u66PhcQQbAHwTog2AJwqKSzVlbro+3XZYT9/eV8MGt/N6EgAENaINQJU7WVQWbJ9tLwu278cSbABwJkQbgCp1sqhUk+akacWOI/rjHf10+6C2Xk8CgJBAtAGoMieKSjRxdppSdx7VM8P66dYBBBsAVBTRBqBK5BeWaMLsNKXvOqpnhvXXLQPaeD0JAEIK0QYg4PILSzRhVprSdx/Vn+/sr5v7E2wAcLaINgABlVdYogmzUrVqT7b+OnyAbuzX2utJABCSiDYAAZNbUKzxs9K0JiNbzw4foKF9W3k9CQBCFtEGICCOFxRrfFKq1mXm6G8jBui6PgQbAJwPog1ApTteUKyxM1O1YW+O/jZygK7tTbABwPki2gBUqpyTxRqblKqN+3L0/KiB+l6vll5PAoCwQLQBqDQ5J4o1JilFm/Yf1wujBunqni28ngQAYYNoA1Apck4Ua/TMFG0+kKuXRg/SlT0INgCoTEQbgPOWfaJIo2emaMuBPL00ZqCu6E6wAUBlI9oAnJdj+UUaNSNF27Ly9PLYQbq8W3OvJwFAWCLaAJyzo/5g256Vp8Sxsbq0azOvJwFA2KoWyBs3s2vNbLOZbTOzx05zeXsz+9jMVpvZOjO73n9+BzM7aWZr/F8vlbvOIDNb77/NZ83MAnkfAJzekbxCjUxM1o6sPM0g2AAg4AIWbWYWJel5SddJ6ilphJn1POXHHpe01Dk3QNJwSS+Uu2y7c66//+uucue/KGmKpC7+r2sDdR8AnN7hvEKNTEzRzsP5mjlusL5LsAFAwAXySFucpG3OuR3OuSJJiyXdfMrPOEn1/d83kLTv227QzFpJqu+cS3bOOUlzJd1SubMBfJus3EKNmJ6s3UfzlTR+sC7u0tTrSQAQEQIZbW0kZZQ7nek/r7wnJI02s0xJ70m6r9xlHf0Pm/7bzC4pd5uZZ7hNSZKZTTWzdDNLz8rKOo+7AeArh3ILNCIxWZnHTipp/GBd1JlgA4CqEtDntFXACEmznXNtJV0vaZ6ZVZO0X1J7/8OmD0taaGb1v+V2/odzbrpzLtY5F9usGQ/dAOfr0PECjZierL3HTmrWhMH6TieCDQCqUiBfPbpXUrtyp9v6zytvkvzPSXPOrTCzGElNnXOHJBX6z19pZtsldfVfv+0ZbhNAJTt0vEDDE5N1IKdAsycMVvyFTbyeBAARJ5BH2tIkdTGzjmZWQ2UvNHj7lJ/ZI+lKSTKzHpJiJGWZWTP/CxlkZheq7AUHO5xz+yUdN7ME/6tGx0p6K4D3AYh4B3IKNHx6sg7mFGjOxDiCDQA8ErAjbc65EjO7V9KHkqIkJTnnvjCzJyWlO+felvSIpEQze0hlL0oY75xzZvZdSU+aWbEkn6S7nHNH/Td9t6TZkmpJet//BSAA9uec1IjpyTqcV6Q5E+MU26Gx15MAIGJZ2Ysww1tsbKxLT0/3egYQUvZln9SIxGQd8QfboAsaeT0JACKCma10zsWeej6fiADgf+zNLjvCdiy/SHMnxWlge4INALxGtAH4msxjJzQiMVnZJ4o1b3K8+rdr6PUkAICINgDlZBwtC7ack8WaPyle/Qg2AAgaRBsASWXBNnx6snILirVgcrz6tiXYACCYEG0AtOdI2RG2vMISLZySoN5tGng9CQBwCqINiHC7j+RrxPRknSgu1YLJ8QQbAAQpog2IYLsO52tEYrIKiku1cHKCerY+q0+LAwBUIaINiFA7D+dr+PQVKi51WjglQT1aEWwAEMyINiACbc/K04jpySr1OS2akqBuLet5PQkAcAZEGxBhth3K08jEZPmc06KpCeragmADgFBAtAERZNuhXA2fniJJWjQlQV0INgAIGUQbECG2HszViMRkSabFU+PVuTnBBgChpJrXAwAE3paDuRo+PVnVzLR4agLBBgAhiCNtQJj76jlsUdVMi6YmqFOzul5PAgCcA460AWFs5+F8jfQ/JEqwAUBoI9qAMJVx9IRGJiarxOe0cEo8wQYAIY5oA8LQ3uyTGj49WSeKSjV/Ujxv6wEAYYBoA8LMgZwCjUxM1vGCYs2fFM9HUwFAmCDagDByKLcs2I7kFWnuxDj1acuHvwNAuCDagDBxJK9QoxJTdOB4gWZNGKwB7Rt5PQkAUImINiAMHMsv0qgZKco4dkIzxw3W4A6NvZ4EAKhkvE8bEOJyThZrTFKKdhzO18xxsRrSqYnXkwAAAcCRNiCE5RYUa2xSqrYcyNPLYwbpki7NvJ4EAAgQog0IUfmFJRo/K01f7M3R86MG6vJuzb2eBAAIIB4eBULQyaJSTZydpjUZ2frbiAG6umcLrycBAAKMI21AiCkoLtWUuelK23VUzwzrp+v6tPJ6EgCgCnCkDQghhSWlumv+Sn22/bD+cEc/3dy/jdeTAABVhCNtQIgoKvHpngWrtXxzln57ax/dMait15MAAFWIaANCQEmpTw8sXq1lmw7qqZt7aXhce68nAQCqGNEGBLlSn9NDS9fq/Q0H9PMbemrMkA5eTwIAeIBoA4KYz+f0w1fX6u9r9+mx67pr0sUdvZ4EAPAI0QYEKZ/P6advrNfrq/bq4au76q5LO3k9CQDgIaINCELOOf3y7S+0OC1D913RWfdf2cXrSQAAjxFtQJBxzumpdzZpXvJuTbv0Qj18dVevJwEAggDRBgQR55x+98GXSvpspyZc1EGPXdtdZub1LABAECDagCDy54+26OV/79DohPb6xQ09CTYAwH8RbUCQeO6fW/Xsv7Zp+OB2evKm3gQbAOBriDYgCLz07+3600dbdNvANvrNrX1UrRrBBgD4OqIN8NjMT3fqd+9/qRv7tdYf7uhHsAEATotoAzw0b8UuPfXORl3Xu6WeGdZPUQQbAOAbEG2ARxan7tHP3/pCV/Vorr8OH6DoKP5xBAB8M/6UADzw2spM/eSN9bq0azM9P2qgalTnH0UAwLfjTwqgir29dp9++OpaXdSpqV4eM0g1q0d5PQkAEAKINqAKvb9+vx5askaDOzRW4thYxUQTbACAiiHagCry0caDum/Rag1o11BJ4werVg2CDQBQcUQbUAU+3nxI9yxYpV5tGmjWhMGqU7O615MAACGGaAMC7NOthzVt3kp1bVlXcyfEqV5MtNeTAAAhiGgDAih5xxFNnpumC5vW0byJ8WpQm2ADAJwbog0IkPRdRzVxdpraNaqt+ZPj1ahODa8nAQBCGNEGBMCajGyNn5WmlvVjtGBKvJrWren1JABAiCPagEq2YW+OxsxMUZO6NbRwSoKa14vxehIAIAwQbUAl2rT/uEbPTFH9mGgtnJKglg0INgBA5SDagEqy9WCuRs1IUa3oKC2akqA2DWt5PQkAEEaINqASbM/K04jEFFWvZlo4JUHtm9T2ehIAIMwQbcB52nU4XyMTkyU5LZySoI5N63g9CQAQhog24DxkHD2hkYnJKirxacHkBHVuXtfrSQCAMMVn6QDnaF/2SY2ckaz8olItnBKvbi3reT0JABDGONIGnIODxws0MjFZ2fnFmjcpTr1aN/B6EgAgzBFtwFnKyi3UyMRkZeUWas6kOPVt29DrSQCACMDDo8BZOJpfpNEzUrQvu0BzJsZpYPtGXk8CAEQIjrQBFZR9oizYdh3J18xxsYrr2NjrSQCACEK0ARVwvKBYY5NSte1QnhLHxuo7nZt6PQkAEGGINuAM8gpLNC4pVZv2H9dLYwbqu12beT0JABCBiDbgW5woKtGEWalan5mjv40cqCu6t/B6EgAgQhFtwDc4WVSqSbPTtXL3Mf11+AB9r1dLrycBACIYrx4FTqOguFRT56UreecR/eXO/hrat5XXkwAAEY4jbcApikp8unvBKn2y9bCevr2vbu7fxutJAAAQbUB5xaU+3btwlf715SH95tY++n5sO68nAQAgiWgD/quk1KcHl6zRPzYe1K9u6qWR8e29ngQAwH8RbYCkUp/To6+s1bvr9uvxoT007jsdvJ4EAMDXEG2IeKU+px+/tk5vrtmnH13bTZMvudDrSQAA/A9ePYqIVupz+uEra/X66r166Kquuvuyzl5PAgDgtIg2RKySUp8eXrpWb6/dp0eu7qr7ruzi9SQAAL4R0YaIVFzq0wOLV+u99Qf02HXdddelnbyeBADAtyLaEHGKSsre1uMfGw/q8aE9eA4bACAkEG2IKIUlpbp7/ir988tD+tVNvXiVKAAgZBBtiBgFxaWaNm+l/r0lS/93a2+Nir/A60kAAFQY0YaIcLKoVJPnpunz7Uf09O19NWwwn3QAAAgtRBvCXn5hiSbOTlParqP60/f76baBbb2eBADAWSPaENZyC4o1YVaaVmdk68939ufD3wEAIYtoQ9g6XlCscUmpWp+Zo+dGDND1fVp5PQkAgHNGtCEs5Zwo1pikFG3af1zPjxqo7/Vq6fUkAADOC9GGsHMsv0ijZqRo26E8vTR6kK7s0cLrSQAAnDeiDWHlcF6hRs9I0c7D+UocF6tLuzbzehIAAJWCaEPYOJRboFGJKco4dkJJ4wfros5NvZ4EAEClIdoQFg4eL9CIxGQdyCnQ7AlxSriwideTAACoVEQbQt6+7JMamZisrNxCzZkYp8EdGns9CQCASke0IaRlHD2hkTOSlZ1frHmT4zWwfSOvJwEAEBBEG0LWniMnNCIxWbkFxVowJV592zb0ehIAAAFDtCEk7TycrxHTk1VYUqqFUxLUu00DrycBABBQRBtCzrZDeRqZmKxSn9PCKQnq0aq+15MAAAg4og0hZcvBXI1MTJZkWjw1QV1a1PN6EgAAVYJoQ8jYuO/4/2vvvuOkKu89jn9/uywsHZESpC0iIh0EFoIhMYHEQixJ8IqRXjSJxpbE9tJEvUluYhJrjMbQm2AUr8YYYuyXxGyhI0U6LB0EpEjZ3d/9Y8fXixCCu8vMPnNmPu+/ds4czn7n9Swz33nOmXk0dEKesjJNM8f1VdvGdUJHAgCgylDaEAnLtuzX0Al5qpmVqefG9VVOo9qhIwEAUKUobUh6izfv07AJeaqbnaVZN/RVy4a1QkcCAKDKZYQOAJzO/I17NXR8nhrUqq7ZN1LYAADpi5k2JK389R9p1KR8NamXrZnj+qhZ/ZqhIwEAEAwzbUhK76/doxET8/W5+tmadUNfChsAIO0x04akM2/1bo2dWqBWDWtpxti+aly3RuhIAAAEx0wbkso7q3Zq9JQC5ZxdW8+No7ABAPApZtqQNN5YvkPfm7FA7ZrW0fQxfXRW7eqhIwEAkDSYaUNSmLtsu74zfb46NKurmWP7UtgAADgJM20I7tUlW3XrrEXq1qK+Jo/OVb3srNCRAABIOpQ2BPXyoi26ffYi9Wx9liaNylWdGvxJAgBwKgk9PWpml5rZKjNbY2Z3n+L+Vmb2tpktNLMlZnb5Ke4/aGY/PGHbBjNbamaLzKwwkfmRWC/ML9Ltsxcpt01DTRlNYQMA4HQS9ippZpmSnpL0VUlFkgrM7BV3X37CbvdJet7dnzazjpJek5Rzwv2PSPrLKQ7/ZXffnZjkqAqzCzbp7jlLdVHbRvrD8F6qWT0zdCQAAJJaImfaciWtcfd17n5M0ixJV520j0uqF/u5vqStn95hZldLWi/pgwRmRADT/7lRd724VF9s11jjR1DYAAAoj0SWtuaSNp9wuyi27UQPSBpqZkUqm2X7viSZWR1Jd0l68BTHdUmvm9l8M7vhP/1yM7vBzArNrHDXrl2VfxSIq8l/X6/7/neZBnZoomeH91R2FoUNAIDyCP2VH9dJmuzuLSRdLmmamWWorMw96u4HT/FvvuDuF0q6TNJNZvbFUx3Y3Z91917u3qtx48YJio+KGP9/6/TAn5brkk5N9bvre6pGNQobAADllcgrv7dIannC7RaxbScaI+lSSXL3980sW1IjSX0kDTazhyU1kFRqZkfc/bfuviW2/04ze0llp2HfS+DjQBz87p01enjuKg3q0kyPDemurMzQ7xcAAIiWRL5yFkhqZ2ZtzKy6pCGSXjlpn02SBkiSmXWQlC1pl7v3d/ccd8+R9Jikn7v7b82stpnVje1fW9LXJC1L4GNAHDz+xmo9PHeVrup+jh6nsAEAUCkJm2lz92Izu1nSXyVlSpro7h+Y2UOSCt39FUk/kPQHM7tdZdeqjXR3P81hm0p6ycw+zT7T3ecm6jHgzLi7Hvnbh3ryrTX61oUt9PDgrsrMsNCxAACIJDt9R0oNvXr18sJCvtKtKrm7fjl3lZ55d62G9G6pn3+jizIobAAAfCYzm+/uvU7ezreZIu7cXT/78wqNn7deQ/u20kNXdqawAQBwhihtiCt314N/o6AKrwAADWpJREFUWq7J/9igkf1y9JMrOip2OhsAAJwBShviprTUdf/LyzQjb5PG9W+jey/vQGEDACBOKG2Ii5JS1z1zluj5wiJ99+K2uvOS9hQ2AADiiNKGM1ZS6vrRHxdrzsItumVAO90+sB2FDQCAOKO04YwUl5TqjucX65XFW3XHV8/XLQPahY4EAEBKorSh0o6XlOrWWQv12tLtuuvSC/Tdi9uGjgQAQMqitKFSjhWX6uaZC/T68h26b1AHje1/buhIAACkNEobKuxocYm+N32B3ly5Uw9e2Ukj+uWEjgQAQMqjtKFCjhwv0Y3T5uvdD3fpZ9/orOv7tA4dCQCAtEBpQ7l9cqxEY6cW6B9r9+jhb3XVf/VuGToSAABpg9KGcjl0tFijJxeoYMNH+vXgbvpWzxahIwEAkFYobfhMB48Wa9SkfC3YtE+PXttdV3VvHjoSAABph9KG0zpyvETjphRqwaZ9emJIDw3q2ix0JAAA0lJG6ABIXsUlpbrluYV6f90e/eaabhQ2AAACorThlEpLXXfPWarXl+/QA1d01NU9OCUKAEBIlDb8G3fXz19boRfmF+m2ge008qI2oSMBAJD2KG34N797Z63Gz1uvkf1ydCtriQIAkBQobfgX0/65Ub/66yp9o0dz/fjrHWVmoSMBAABR2nCCVxZv1Y9fXqaBHZro4cFdlZFBYQMAIFlQ2iBJenvVTt0xe5F65zTUb799obIy+dMAACCZ8MoMFW74SN+dPl/tP1dX40f0UnZWZuhIAADgJJS2NLd868caNblA59SvqSmjc1UvOyt0JAAAcAqUtjS2YfchDZ+Yrzo1qmnqmFw1qlMjdCQAAPAfUNrS1Pb9RzR0Qp5KSks1bUyuWpxVK3QkAABwGpS2NLTv8DENn5invYeOacroXJ3XpG7oSAAA4DOwYHyaOXS0WCMnFWjDnsOaPKq3urZoEDoSAAAoB2ba0sjR4hLdOG2+lhTt05PX9VC/to1CRwIAAOXETFuaKCl13TZrkeat2a1fDe6qSzp9LnQkAABQAcy0pQF3171zluovy7brvkEddE2vlqEjAQCACqK0pYFfzF2p2YWbdfOXz9PY/ueGjgMAACqB0pbinn5nrX7/7joN7dtKP/ja+aHjAACASqK0pbDn8jfpl3NX6opu5+ihKzvLjAXgAQCIKkpbivrzkm2696Wlurh9Y/3mmm7KyKCwAQAQZZS2FPTeh7t02+yF6tnqLD19fU9Vr8YwAwAQdbyap5j5G/fqxmnz1bZxHU0Y2Vs1q2eGjgQAAOKA0pZCVm0/oNGTC9SkXg1NHZOr+jWzQkcCAABxQmlLEZv2HNawCXnKzsrQ9DF91KRuduhIAAAgjlgRIQXs/PiIhk7I07GSUj1/4+fVsmGt0JEAAECcMdMWcfsPH9fwifnaffCoJo3srfOb1g0dCQAAJAClLcIOHyvW6CkFWrfrkJ4d1ks9Wp0VOhIAAEgQSltEHSsu1XemL9DCTXv1+JDu+kK7RqEjAQCABOKatggqKXXd8fwivffhLv3im110WZdmoSMBAIAEY6YtYtxd97+8TK8u2aZ7LrtAQ3JbhY4EAACqAKUtYn79+irNzNuk73yprW78UtvQcQAAQBWhtEXIH95bp6feXqvrclvqrkvbh44DAACqEKUtIp4v3KyfvbZCg7o000+v7iIzFoAHACCdUNoiYO6y7br7xSXq366RHrm2mzIzKGwAAKQbSluS+/ua3brluYXq1rKBnhnaUzWqsQA8AADpiNKWxBZt3qdxUwvVplFtTRrZW7Vr8A0tAACkK0pbklq944BGTsrX2XWqa+qYXDWoVT10JAAAEBClLQkV7T2sYRPylZWZoelj+qhpvezQkQAAQGCUtiSz68BRDZuQr8PHijV1dK5an107dCQAAJAEuEgqiXx85LhGTMzXtv2faMbYPurQrF7oSAAAIEkw05YkPjlWorGTC7V65wE9M7SnerZuGDoSAABIIsy0JYHjJaW6aeYCFWz8SE8M6aGL2zcJHQkAACQZZtoCKy11/fCPi/XWyp366dWddUW3c0JHAgAASYjSFpC764E/faCXF23Vjy5pr+v7tA4dCQAAJClKW0CPvrFaU9/fqHH92+h7F7cNHQcAACQxSlsgE+et1xNvrtY1PVvo3ss7sAA8AAA4LUpbAC/OL9JDry7XJZ2a6n++2YXCBgAAPhOlrYr9bfkO3fniEvVre7YeH9JD1TIZAgAA8NloDFXo/bV7dNPMBep8Tj09O7yXsrMyQ0cCAAARQWmrIkuL9mvc1EK1alhLk0blqk4NviIPAACUH6WtCqzZeVAjJuWrfs0sTRuTq4a1q4eOBAAAIobSlmBb9n2i4RPylGHStDG5ala/ZuhIAAAggihtCbTn4FENm5CnA0eKNXlUrs5tXCd0JAAAEFFcWJUgB44c14hJ+dqy9xNNG9NHnZvXDx0JAABEGDNtCXDkeInGTinUym0H9PTQC5XbpmHoSAAAIOKYaYuz4pJS3TxzofLWf6THru2ur1zQNHQkAACQAphpi6PSUtedLy7RGyt26MErO+nqHs1DRwIAACmC0hYn7q7//vNyzVmwRbcPPF8j+uWEjgQAAFIIpS1OnnxrjSb9fYNGXZSjWwacFzoOAABIMZS2OJj6/gY98rcP9c0ezXX/oI4sAA8AAOKO0hYHq3cc1MAOTfTLwV2VkUFhAwAA8cenR+Pgoas66XiJKyuTDgwAABKD0hYHZqbq1ZhhAwAAicPUEAAAQARQ2gAAACKA0gYAABABlDYAAIAIoLQBAABEAKUNAAAgAihtAAAAEUBpAwAAiABKGwAAQARQ2gAAACKA0gYAABABlDYAAIAIoLQBAABEAKUNAAAgAihtAAAAEUBpAwAAiABKGwAAQARQ2gAAACKA0gYAABABlDYAAIAIoLQBAABEAKUNAAAgAszdQ2dIODPbJWljgn9NI0m7E/w7kFiMYfQxhtHG+EUfYxgfrd298ckb06K0VQUzK3T3XqFzoPIYw+hjDKON8Ys+xjCxOD0KAAAQAZQ2AACACKC0xc+zoQPgjDGG0ccYRhvjF32MYQJxTRsAAEAEMNMGAAAQAZQ2AACACKC0xYGZXWpmq8xsjZndHToPys/MWprZ22a23Mw+MLNbQ2dC5ZhZppktNLNXQ2dBxZlZAzN7wcxWmtkKM/t86EwoPzO7PfYcuszMnjOz7NCZUhGl7QyZWaakpyRdJqmjpOvMrGPYVKiAYkk/cPeOkvpKuonxi6xbJa0IHQKV9rikue5+gaRuYiwjw8yaS7pFUi937ywpU9KQsKlSE6XtzOVKWuPu69z9mKRZkq4KnAnl5O7b3H1B7OcDKnuhaB42FSrKzFpIGiRpfOgsqDgzqy/pi5ImSJK7H3P3fWFToYKqSappZtUk1ZK0NXCelERpO3PNJW0+4XaReNGPJDPLkdRDUl7YJKiExyTdKak0dBBUShtJuyRNip3iHm9mtUOHQvm4+xZJv5a0SdI2Sfvd/fWwqVITpQ2QZGZ1JL0o6TZ3/zh0HpSfmX1d0k53nx86CyqtmqQLJT3t7j0kHZLE9cERYWZnqewMUxtJ50iqbWZDw6ZKTZS2M7dFUssTbreIbUNEmFmWygrbDHefEzoPKuwiSVea2QaVXZ7wFTObHjYSKqhIUpG7fzrL/YLKShyiYaCk9e6+y92PS5ojqV/gTCmJ0nbmCiS1M7M2ZlZdZRdfvhI4E8rJzExl19GscPdHQudBxbn7Pe7ewt1zVPb/7y13511+hLj7dkmbzax9bNMAScsDRkLFbJLU18xqxZ5TB4gPkiREtdABos7di83sZkl/VdknZia6+weBY6H8LpI0TNJSM1sU23avu78WMBOQjr4vaUbsze86SaMC50E5uXuemb0gaYHKPpG/UCxnlRAsYwUAABABnB4FAACIAEobAABABFDaAAAAIoDSBgAAEAGUNgAAgAigtAFAgpjZxWb2augcAFIDpQ0AACACKG0A0p6ZDTWzfDNbZGa/N7NMMztoZo+a2Qdm9qaZNY7t293M/mlmS8zspdi6izKz88zsDTNbbGYLzKxt7PB1zOwFM1tpZjNi3xgPABVGaQOQ1sysg6RrJV3k7t0llUi6XlJtSYXu3knSu5J+EvsnUyXd5e5dJS09YfsMSU+5ezeVrbu4Lba9h6TbJHWUdK7KVuEAgApjGSsA6W6ApJ6SCmKTYDUl7ZRUKml2bJ/pkuaYWX1JDdz93dj2KZL+aGZ1JTV395ckyd2PSFLsePnuXhS7vUhSjqR5iX9YAFINpQ1AujNJU9z9nn/ZaHb/SftVds2/oyf8XCKedwFUEqdHAaS7NyUNNrMmkmRmDc2stcqeHwfH9vm2pHnuvl/SXjPrH9s+TNK77n5AUpGZXR07Rg0zq1WljwJAyuMdH4C05u7Lzew+Sa+bWYak45JuknRIUm7svp0qu+5NkkZIeiZWytZJGhXbPkzS783sodgxrqnChwEgDZh7ZWf8ASB1mdlBd68TOgcAfIrTowAAABHATBsAAEAEMNMGAAAQAZQ2AACACKC0AQAARAClDQAAIAIobQAAABHw/x1VfB24xnWcAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(10, 10))\n",
        "plt.plot(history.history['loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train'], loc='upper left')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "9dIpZk6AK8Zl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 621
        },
        "outputId": "e192b915-2500-46e8-8b09-1af7e7d7fc3c"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 720x720 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAJcCAYAAAC8DwN/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hW5f3H8c+dTRYBEvYUZO+EBNBarbMuVGRPWeKobe3P1rbWVluttW5lywZF3LZ1VOqohTDClL2RMMPIIHvcvz8SbKSoAfLkfsb7dV25fPKcc5JPvLziJ+ec+3uMtVYAAADwDkGuAwAAAOC/KGcAAABehHIGAADgRShnAAAAXoRyBgAA4EUoZwAAAF6EcgYgoBlj5hhj/lTFffcaY6660K8DAN+FcgYAAOBFKGcAAABehHIGwOtVXE58wBizwRiTa4yZaYxpYIz5wBiTY4xZYoypU2n/m40xm4wxmcaYz4wxHSpt62GMWVNx3GuSIs74XjcaY9ZVHLvMGNP1PDOPN8bsNMacMMa8Z4xpXPG+McY8a4w5aozJNsZ8aYzpXLHtemPM5opsB4wx/3de/8IA+DTKGQBf0V/S1ZLaSrpJ0geSfiMpQeW/y+6TJGNMW0mvSvpZxbb3Jf3NGBNmjAmT9I6k+ZLqSnq94uuq4tgekmZJulNSPUnTJL1njAk/l6DGmB9J+rOkgZIaSdonaVHF5mskXVbxc9Su2Od4xbaZku601sZI6izpk3P5vgD8A+UMgK940Vp7xFp7QNIXklZYa9daawskvS2pR8V+gyT9w1r7sbW2WNJTkmpJ6iupt6RQSc9Za4uttW9IWlXpe0yQNM1au8JaW2qtnSupsOK4czFM0ixr7RprbaGkX0vqY4xpKalYUoyk9pKMtXaLtfZQxXHFkjoaY2KttSettWvO8fsC8AOUMwC+4kil1/ln+Ty64nVjlZ+pkiRZa8sk7ZfUpGLbAWutrXTsvkqvW0j6RcUlzUxjTKakZhXHnYszM5xS+dmxJtbaTyS9JGmSpKPGmOnGmNiKXftLul7SPmPM58aYPuf4fQH4AcoZAH9zUOUlS1L5PV4qL1gHJB2S1KTivdOaV3q9X9Jj1tq4Sh+R1tpXLzBDlMovkx6QJGvtC9baREkdVX5584GK91dZa/tJqq/yy6+Lz/H7AvADlDMA/maxpBuMMVcaY0Il/ULllyaXSUqVVCLpPmNMqDHmNknJlY6dIWmiMSal4sb9KGPMDcaYmHPM8KqkO4wx3SvuV3tc5Zdh9xpjelV8/VBJuZIKJJVV3BM3zBhTu+JybLaksgv49wDAR1HOAPgVa+02ScMlvSjpmMoXD9xkrS2y1hZJuk3SaEknVH5/2luVjk2TNF7llx1PStpZse+5Zlgi6XeS3lT52brWkgZXbI5VeQk8qfJLn8cl/bVi2whJe40x2ZImqvzeNQABxnzz1gsAAAC4xJkzAAAAL0I5AwAA8CKUMwAAAC9COQMAAPAiIa4DVJf4+HjbsmVL1zEAAAC+1+rVq49ZaxPOts1vylnLli2VlpbmOgYAAMD3Msbs+7ZtXNYEAADwIpQzAAAAL0I5AwAA8CJ+c8/Z2RQXFys9PV0FBQWuo3hcRESEmjZtqtDQUNdRAADABfDrcpaenq6YmBi1bNlSxhjXcTzGWqvjx48rPT1drVq1ch0HAABcAL++rFlQUKB69er5dTGTJGOM6tWrFxBnCAEA8Hd+Xc4k+X0xOy1Qfk4AAPyd35czAAAAX0I587DMzExNnjz5nI+7/vrrlZmZ6YFEAADAm1HOPOzbyllJScl3Hvf+++8rLi7OU7EAAICX8uvVmt7gwQcf1K5du9S9e3eFhoYqIiJCderU0datW7V9+3bdcsst2r9/vwoKCvTTn/5UEyZMkPTfx1GdOnVKP/7xj3XppZdq2bJlatKkid59913VqlXL8U8GAAA8IWDK2SN/26TNB7Or9Wt2bByr39/U6Tv3eeKJJ7Rx40atW7dOn332mW644QZt3Ljx65EXs2bNUt26dZWfn69evXqpf//+qlev3je+xo4dO/Tqq69qxowZGjhwoN58800NHz68Wn8WAADgHQKmnHmL5OTkb8wie+GFF/T2229Lkvbv368dO3b8Tzlr1aqVunfvLklKTEzU3r17aywvAACoWQFTzr7vDFdNiYqK+vr1Z599piVLlig1NVWRkZG6/PLLzzqrLDw8/OvXwcHBys/Pr5GsAACg5rEgwMNiYmKUk5Nz1m1ZWVmqU6eOIiMjtXXrVi1fvryG0wEAAG8TMGfOXKlXr54uueQSde7cWbVq1VKDBg2+3nbddddp6tSp6tChg9q1a6fevXs7TAoAALyBsda6zlAtkpKSbFpa2jfe27Jlizp06OAoUc0LtJ8XAABfZYxZba1NOts2LmsCAAB4EcoZAACAF/H7cuYvl22/T6D8nAAA+Du/LmcRERE6fvy43xcXa62OHz+uiIgI11EAAMAF8uvVmk2bNlV6eroyMjKq5etZa2WMqZavVd0iIiLUtGlT1zEAAMAF8utyFhoa+o1p/Bciu6BYo2at1A1dGmncDy6qlq8JAABwJr++rFmdosNC1DA2Qo+/v0X/3l49Z+IAAADORDmroqAgo6cGdFPbBjG695U12nMs13UkAADghyhn5yAqPEQzRiYpOMho/Lw05RQUu44EAAD8DOXsHDWrG6lJw3pqz7Fc/fy1dSor8++VoAAAoGZRzs5D39bx+v1NHbVky1E98/F213EAAIAf8evVmp40oncLbT6YrZc+3an2jWJ0Y9fGriMBAAA/wJmz82SM0SP9OimxRR098PoGbTqY5ToSAADwA5SzCxAeEqypwxMVFxmqCfNW6/ipQteRAACAj6OcXaCEmHBNG5GoY6cKddfCNSouLXMdCQAA+DDKWTXo2jROT97eVSv3nNAjf9vkOg4AAPBhLAioJv26N9HmQ9ma9vludWgUq2EpLVxHAgAAPogzZ9Xol9e21+XtEvT7dzdp5Z4TruMAAAAfRDmrRsFBRs8P7qHmdSN114LVOpCZ7zoSAADwMZSzala7Vqimj0xSUUmZJsxLU35RqetIAADAh1DOPKBN/Wi9MKSHNh/K1gNvrJe1POIJAABUDeXMQ65oX1+/vLa9/r7hkKZ8vst1HAAA4CMoZx408YcX6aZujfXXj7bpX1uOuI4DAAB8AOXMg4wxerJ/V3VqHKufLlqnnUdzXEcCAABejnLmYbXCgjVtRJIiQoM0ft5qZeUXu44EAAC8GOWsBjSJq6UpwxOVfjJP9726VqVlLBAAAABnRzmrIb1a1tWj/Trr8+0ZevLDra7jAAAAL8Xjm2rQkOTm2nwwW9P+vVvtG8Xo1h5NXUcCAABehjNnNezhmzoqpVVd/erNL7UhPdN1HAAA4GUoZzUsNDhIk4f1VEJ0uCbMW62jOQWuIwEAAC9COXOgXnS4po9MVFZ+sSbOX63CEh7xBAAAylHOHOnUuLaeGtBNa77K1MPvbOIRTwAAQBLlzKkbujbST37URq+l7de81H2u4wAAAC9AOXPs51e11VUdGujRv2/Wsp3HXMcBAACOUc4cCwoyenZQN10UH6W7X1mj/SfyXEcCAAAOUc68QExEqGaMTFJZmdX4eWnKLSxxHQkAADhCOfMSLeOj9NLQntp+JEf3L16nMh7xBABAQKKceZHL2iboN9d30EebjujFT3a6jgMAABzg8U1eZuylrbT5ULaeXbJd7RrG6LrODV1HAgAANYgzZ17GGKPHb+2ibs3idP/iddp6ONt1JAAAUIMoZ14oIjRY00ckKjo8ROPnpelkbpHrSAAAoIZQzrxUg9gITR2RqCNZhbrnlTUqKS1zHQkAANQAypkX69m8jh6/rYuW7Tqux97f4joOAACoASwI8HK3JzbV5oPZmrV0jzo0itXApGauIwEAAA/izJkP+M317XVpm3g99PZGrd530nUcAADgQZQzHxASHKSXhvZQw9oRmrhgtQ5nFbiOBAAAPIRy5iPiIsP08qgk5RWW6M75aSooLnUdCQAAeADlzIe0bRCjZwd11/r0LP36rS9lLY94AgDA31DOfMw1nRrq/qvb6u21B/TyF3tcxwEAANWMcuaD7r2ijX7cuaH+/MEWfb49w3UcAABQjShnPigoyOipAd3UtkGMfvLKGu05lus6EgAAqCaUMx8VFR6iGSOTFBxkNH5emnIKil1HAgAA1YBy5sOa1Y3UpGE9tedYrn62aJ3KylggAACAr6Oc+bi+reP1+5s66l9bj+rpj7e5jgMAAC4Qj2/yAyN6t9Dmg9ma9OkutW8Yq5u6NXYdCQAAnCfOnPkBY4we6ddJiS3q6IE31mvTwSzXkQAAwHminPmJ8JBgTR2eqDqRYZowb7WOnSp0HQkAAJwHypkfSYgJ1/QRSTp2qlB3L1ijopIy15EAAMA5opz5mS5Na+vJ27tq5d4TeuRvm1zHAQAA54gFAX6oX/cm2nwoW9M+360OjWI1vHcL15EAAEAVcebMT/3y2va6vF2C/vDeJq3cc8J1HAAAUEWUMz8VHGT0/OAeal43UnctWK30k3muIwEAgCqgnPmx2rVCNX1kkopKyjRh3mrlF5W6jgQAAL4H5czPtakfrReG9NCWw9l64I31spZHPAEA4M0oZwHgivb19ctr2+vvGw5p8me7XMcBAADfgXIWICb+8CLd3K2xnvrnNv1ryxHXcQAAwLegnAUIY4z+0r+rOjWO1U8XrdPOozmuIwEAgLOgnAWQWmHBmjYiSRGhQRo/b7Wy8opdRwIAAGegnAWYJnG1NGV4otJP5ukni9aqtIwFAgAAeBPKWQDq1bKuHu3XWf/enqG/fLjVdRwAAFAJj28KUEOSm2vzwWxN//dudWgUo1t7NHUdCQAAiDNnAe3hmzoqpVVd/erNL7V+f6brOAAAQJSzgBYaHKTJw3oqITpcE+an6Wh2getIAAAEPMpZgKsXHa4ZI5OUnV+iiQtWq7CERzwBAOAS5Qzq2DhWTw3opjVfZeqhtzfyiCcAABxiQQAkSTd0baRth9vohU92qlndSN135cWuIwEAEJAoZ/jaz69uq/TMfD3z8XY1jI3QwF7NXEcCACDgUM7wtdOPeMrIKdSv3/5SCbHhuqJdfdexAAAIKNxzhm8IDQ7SlOGJat8wRncvWMOIDQAAaphHy5kx5jpjzDZjzE5jzINn2d7CGPMvY8wGY8xnxpimlbaNMsbsqPgY5cmc+Kbo8BDNvqOX6kWHacycVdp3PNd1JAAAAobHypkxJljSJEk/ltRR0hBjTMczdntK0jxrbVdJj0r6c8WxdSX9XlKKpGRJvzfG1PFUVvyv+jERmjsmWaXWatSslTp+qtB1JAAAAoInz5wlS9pprd1trS2StEhSvzP26Sjpk4rXn1bafq2kj621J6y1JyV9LOk6D2bFWbROiNbMUb10KKtAY+amKa+oxHUkAAD8nifLWRNJ+yt9nl7xXmXrJd1W8fpWSTHGmHpVPFbGmAnGmDRjTFpGRka1Bcd/JbaooxeG9NCX6Zm695W1Kiktcx0JAAC/5npBwP9J+qExZq2kH0o6IKnKI+qttdOttUnW2qSEhARPZQx413ZqqEf6ddYnW4/qd+8ypBYAAE/y5CiNA5IqD8pqWvHe16y1B1Vx5swYEy2pv7U20xhzQNLlZxz7mQez4nuM6N1Ch7PyNenTXWoYW0s/vYohtQAAeIInz5ytknSxMaaVMSZM0mBJ71XewRgTb4w5neHXkmZVvP5I0jXGmDoVCwGuqXgPDv3fNe10W88menbJdi1etf/7DwAAAOfMY+XMWlsi6V6Vl6otkhZbazcZYx41xtxcsdvlkrYZY7ZLaiDpsYpjT0j6o8oL3ipJj1a8B4dOD6n9wcXx+vXbX+rTrUddRwIAwO8Yf7l/KCkpyaalpbmOERBOFZZo0LRU7c7I1aIJvdWtWZzrSAAA+BRjzGprbdLZtrleEAAfxJBaAAA8h3KG83J6SG2ZtRo5a6WOMaQWAIBqQTnDeWudEK2XR/XS4awCjZ2ziiG1AABUA8oZLkhiizp6cUgPfXkgiyG1AABUA8oZLtg1nRrq0YohtQ+9w5BaAAAuhCeH0CKADO/dQoezCvTSpzvVsHaEfnZVW9eRAADwSZQzVJtfXNNWh7IK9NySHWpUO0KDejV3HQkAAJ9DOUO1Mcboif5dlHGqUL95e6MSYsL1o/YNXMcCAMCncM8ZqlVocJAmD+upDo1idM/CtVq/P9N1JAAAfArlDNUuOjxEs0b/d0jt3mMMqQUAoKooZ/CIykNqR81mSC0AAFVFOYPHtE6I1szRvXQkmyG1AABUFeUMHtWzeR29OKSnvjyQpXsWrmFILQAA34NyBo+7umMD/fGWzvp0W4Z++zZDagEA+C6M0kCNGJZSPqT2xU92qlEcQ2oBAPg2lDPUmPuv/u+Q2oaxERqczJBaAADORDlDjTHG6M+3ddHRnEL99p2Nqh/LkFoAAM7EPWeoUaHBQZoyrKc6NorVPQvXah1DagEA+AbKGWpcVMWQ2vgYhtQCAHAmyhmcSIgJ19w7kmUZUgsAwDdQzuDMRZWG1I6Zs0q5hQypBQCAcganTg+p3XggS/e8skbFDKkFAAQ4yhmcOz2k9rNtGfrt218ypBYAENAYpQGv8I0htbVr6edXM6QWABCYKGfwGqeH1D7/rx1qWDtCQxhSCwAIQJQzeI3TQ2ozcgr10DsbVT8mXFd2YEgtACCwcM8ZvEpocJAmnx5S+8oarf3qpOtIAADUKMoZvM7pIbUJMeEaOzdNexhSCwAIIJQzeKWEmHDNG5MiSRo1a6UychhSCwAIDJQzeK1W8VGaOSpJR3MKNHYuQ2oBAIGBcgav1qN5Hb3EkFoAQAChnMHrXdWxgf50SxeG1AIAAgKjNOAThqY01+GsfL3wyU41rF1L9zOkFgDgpyhn8Bk/rxhS+8K/dqhhbISGpjCkFgDgfyhn8BnGGD1+WxcdzSnUQ+98qQaxDKkFAPgf7jmDTzk9pLZT49oMqQUA+CXKGXzO6SG19WMiGFILAPA7lDP4pISYcM0dkyyJIbUAAP9COYPPqjykdswchtQCAPwD5Qw+rUfzOpo0tKc2HWRILQDAP1DO4POu7NBAj91aPqT2N28xpBYA4NsYpQG/MCS5+dcz0BrVjtD917RzHQkAgPNCOYPf+PlVF3/jKQIMqQUA+CLKGfyGMUaP3frfIbX1Y8J1VUeG1AIAfAv3nMGvhAYHadLQnurcpLbufXWN1jCkFgDgYyhn8Dunh9Q2iI3QuLlp2p1xynUkAACqjHIGvxQfHa65d1QMqZ3NkFoAgO+gnMFvtYyP0qzRvXQsp4ghtQAAn0E5g1/r3ixOLw3toU0Hs3T3QobUAgC8H+UMfu/0kNrPt2fo1wypBQB4OUZpICAMSW6uw1kFev5fO5QQE65fXdfedSQAAM6KcoaA8bOrLlbGqUJN+WyX6kSGasJlrV1HAgDgf1DOEDCMMfpjv87Kyi/W4+9vVVytMA3s1cx1LAAAvoFyhoASHGT07MDuys4v1oNvbVBsrRBd17mR61gAAHyNBQEIOGEhQZo2IlHdmsXpvlfXaenOY64jAQDwNcoZAlJkWIhmj+6lVvFRmjAvTev3Z7qOBACAJMoZAlhcZJjmjU1W3egwjZ69UjuP5riOBAAA5QyBrUFshBaMTVFIcJCGv7xS6SfzXEcCAAQ4yhkCXot6UZo3Jll5RSUaMXOljp3iOZwAAHcoZ4CkDo1iNWt0Lx3KyteoWSuVXVDsOhIAIEBRzoAKSS3raurwRG07nKNxc9NUUFzqOhIAIABRzoBKLm9XX88M6q5Ve0/o3ld4UDoAoOZRzoAz3NytsR7t11lLthzVr97YoLIyHpQOAKg5PCEAOIsRvVsoM7dIT3+8XbUjQ/XwjR1ljHEdCwAQAChnwLe490dtdDKvWLOW7lGdyDDdd+XFriMBAAIA5Qz4FsYYPXRDB2XlF+uZj7crLjJUI/u0dB0LAODnKGfAdwgKMvpL/y7Kyi/Ww+9uUu1aoerXvYnrWAAAP8aCAOB7hAQH6aWhPZTSqq5+sXi9Pt161HUkAIAfo5wBVRARGqyXRyWpfaMYTVywWqv2nnAdCQDgpyhnQBXFRIRqzh3JahJXS2PmrNLmg9muIwEA/BDlDDgH8dHhmj8uRdHhIRo5a6X2Hst1HQkA4GcoZ8A5ahJXS/PHJqu0rEzDZ67Q4awC15EAAH6Ecgachzb1YzR3TLJO5hZp5KwVyswrch0JAOAnKGfAeeraNE4zRiZp77E8jZ69SrmFJa4jAQD8AOUMuAB928TrxaE9tCE9UxMXrFZhSanrSAAAH0c5Ay7QtZ0a6on+XfXFjmO6/7X1KuVB6QCAC8ATAoBqMDCpmbLyivXY+1sUWytEj9/ahQelAwDOC+UMqCbjL7tIJ/OKNPmzXYqLDNOvrmvvOhIAwAdRzoBq9MC17ZSZX6wpn+1SnchQTbistetIAAAfQzkDqpExRn/s11lZ+cV6/P2tiqsVpoG9mrmOBQDwIZQzoJoFBxk9O7C7svOL9eBbGxRbK0TXdW7kOhYAwEewWhPwgLCQIE0bkahuzeJ036vrtHTnMdeRAAA+gnIGeEhkWIhmj+6lVvFRmjAvTev3Z7qOBADwAZQzwIPiIsM0b2yy6kaHafTsldp5NMd1JACAl6OcAR7WIDZC88ekKDgoSMNfXqn0k3muIwEAvBjlDKgBLeOjNH9ssnKLSjRi5kodO1XoOhIAwEtRzoAa0qFRrGaP7qVDWfkaNWulsguKXUcCAHghyhlQg5Ja1tWU4YnadjhH4+amqaCYB6UDAL6JcgbUsCva1dfTA7tp1d4TuveVNSouLXMdCQDgRShngAP9ujfRo/06a8mWo/rVGxtUVmZdRwIAeAmeEAA4MqJ3C2XmFunpj7erdmSoHr6xo4wxrmMBAByjnAEO3fujNjqZV6xZS/eoTmSY7rvyYteRAACOUc4Ah4wxeuiGDsrKL9YzH29XnchQjejT0nUsAIBDlDPAsaAgo7/076Ks/GI9/N4mxdYKVb/uTVzHAgA4woIAwAuEBAfppaE9lNyyrn6xeL0+3XrUdSQAgCOUM8BLRIQG6+VRSWrfKEZ3LVytVXtPuI4EAHCAcgZ4kZiIUM25I1mNa9fSmDmrtPlgtutIAIAaRjkDvEx8dLjmj0tRdHiIRs5aqb3Hcl1HAgDUIMoZ4IWaxNXS/LHJKi0r0/CZK3Qku8B1JABADaGcAV6qTf0YzR2TrJO5RRoxc4Uy84pcRwIA1ADKGeDFujaN04yRSdp7LE93zFmlvKIS15EAAB5GOQO8XN828XpxaA+t35+pO+evVmFJqetIAAAPopwBPuDaTg31RP+u+mLHMd3/2nqV8qB0APBbPCEA8BEDk5opK69Yj72/RbG1QvX4rZ15UDoA+CHKGeBDxl92kU7mFWnyZ7tUJzJUv7yuvetIAIBq5tHLmsaY64wx24wxO40xD55le3NjzKfGmLXGmA3GmOsr3m9pjMk3xqyr+JjqyZyAL3ng2nYamtJckz/bpRn/3u06DgCgmnnszJkxJljSJElXS0qXtMoY8561dnOl3R6StNhaO8UY01HS+5JaVmzbZa3t7ql8gK8yxuiP/TorK7/8EmftyFANTGrmOhYAoJp48rJmsqSd1trdkmSMWSSpn6TK5cxKiq14XVvSQQ/mAfxGcJDRswO7Kzu/WA++uUGxEaG6rnND17EAANXAk5c1m0jaX+nz9Ir3KvuDpOHGmHSVnzX7SaVtrSoud35ujPnB2b6BMWaCMSbNGJOWkZFRjdEB7xcWEqRpIxLVrVmc7nt1rZbtPOY6EgCgGrgepTFE0hxrbVNJ10uab4wJknRIUnNrbQ9J90t6xRgTe+bB1trp1toka21SQkJCjQYHvEFkWIhmj+6lVvFRGj8vTev3Z7qOBAC4QJ4sZwckVb4RpmnFe5WNlbRYkqy1qZIiJMVbawuttccr3l8taZekth7MCvisuMgwzRubrLrRYRo9e6V2Hs1xHQkAcAE8Wc5WSbrYGNPKGBMmabCk987Y5ytJV0qSMaaDystZhjEmoWJBgYwxF0m6WBLL0oBv0SA2QgvGpigkOEhDZ6zQroxTriMBAM6Tx8qZtbZE0r2SPpK0ReWrMjcZYx41xtxcsdsvJI03xqyX9Kqk0dZaK+kySRuMMeskvSFporX2hKeyAv6gRb0oLRyXotIyqyHTl2s3BQ0AfJIp70K+LykpyaalpbmOATi37XCOhs5YrpBgo0UT+qhVfJTrSACAMxhjVltrk862zfWCAADVrF3DGC0cn6Li0vIzaHuP5bqOBAA4B5QzwA+1bxirheNSVFhSqiEzlmvfcQoaAPgKyhngpzo0itXCcb2VX1yqIdOX66vjea4jAQCqgHIG+LGOjcvPoOUVl59B23+CggYA3o5yBvi5To1ra8HYFJ0qLNHg6RQ0APB2lDMgAHRuUlsLx6Uop6BYQ2YsV/pJChoAeCvKGRAgygtab2Xnlxe0A5n5riMBAM6CcgYEkC5Na2vBuBRl5hVryPTlOkhBAwCvQzkDAkzXpnFaMDZFJ3OLNHj6ch3KoqABgDehnAEBqFuzOM0bm/x1QTucVeA6EgCgAuUMCFA9mtfR3LHJOn6qSENmUNAAwFtQzoAA1rN5Hc0dk6yj2QUaOmO5jmRT0ADANcoZEOASW9TRvLHJOpJdoCHTl+soBQ0AnKKcAVBii7qaOyZZh7MLNHjGch3NoaABgCuUMwCSpKSWdTXnjmQdzio/g5aRU+g6EgAEJMoZgK8lt6qr2aN76WBm+T1oFDQAqHmUMwDfkHJRPc2+o5fST+Zr2MvLdewUBQ0AahLlDMD/6H1RPc0a3UtfncjTsBkrdJyCBgA1hnIG4Kz6tK6nWaN6ad+JXA17mYIGADWFcgbgW/VtE6+Zo3ppz7HygnYit8h1JADwe5QzAN/pkjMK2kkKGgB4FOUMwPe69OJ4zRiZpF0ZpzTs5RXKzKOgAYCnUM4AVMllbRM0Y2SSdlLQAMCjKGcAquyHbRM0fUSidhw5peEzVygrr9h1JADwO5QzAGCuxpgAACAASURBVOfk8nb1NW1EorYfriho+RQ0AKhOlDMA5+yK9vU1dURPbTuco5EUNACoVpQzAOflR+0baMrwntp8KFsjZ61UdgEFDQCqA+UMwHm7skMDTRmWqM0HszRyJgUNAKoD5QzABbmqYwNNGtpTGw9kadSslcqhoAHABaGcAbhg13RqqJeG9tSX6eUF7VRhietIAOCzKGcAqsV1nRvqpaE9tCE9S6MpaABw3ihnAKrNdZ0b6cUhPbR2f6bumL1SuRQ0ADhnlDMA1erHXRrphcE9tOarTN0xexUFDQDOEeUMQLW7oWsjPT+4u1Z/dVJ3zFmlvCIKGgBUFeUMgEfc2LWxnhvUXWl7T2gMBQ0AqoxyBsBjburWWM8O6q6Ve05o7Jw05ReVuo4EAF6PcgbAo/p1b6JnB3XXij3HNXbuKgoaAHwPyhkAj+vXvYmeHthNqbuPa/y8NBUUU9AA4NtQzgDUiFt7NNVTt3fT0l3HKGgA8B0oZwBqTP/Epvrr7d30n50UNAD4NpQzADXq9sSmerJ/V/1n5zHdOX81BQ0AzkA5A1DjBiQ1019u66rPt2do4oLVKiyhoAHAaZQzAE4M7NVMT9zWRZ9ty9DE+RQ0ADiNcgbAmcHJzfXn27ro020ZumvBGgoaAIhyBsCxIcnN9fitXfTJ1qO6Z+EaFZWUuY4EAE5VqZwZY35qjIk15WYaY9YYY67xdDgAgWFoSnP96ZbOWrLlqO55hYIGILBV9czZGGtttqRrJNWRNELSEx5LBSDgDO/dQn/s10kfbz6ie19Zo+JSChqAwFTVcmYq/nm9pPnW2k2V3gOAajGiT0s9cnMn/ZOCBiCAVbWcrTbG/FPl5ewjY0yMJH5rAqh2o/q21B9u6qiPNh3Rfa+upaABCDhVLWdjJT0oqZe1Nk9SqKQ7PJYKQEAbfUkrPXxjR32w8bB+uoiCBiCwhFRxvz6S1llrc40xwyX1lPS852IBCHRjLm2lMmv1p39skTHr9Pyg7goJZoE5AP9X1d90UyTlGWO6SfqFpF2S5nksFQBIGveDi/TQDR30jw2H9PPF61XCGTQAAaCqZ85KrLXWGNNP0kvW2pnGmLGeDAYAUnlBK7NWj7+/VUbSMwO7cQYNgF+rajnLMcb8WuUjNH5gjAlS+X1nAOBxEy5rLWulP3+wVcZITw+goAHwX1UtZ4MkDVX5vLPDxpjmkv7quVgA8E13/rC1yqz0lw+3qrC4TM8P6a7wkGDXsQCg2lXpT09r7WFJCyXVNsbcKKnAWss9ZwBq1F2Xt9bvb+qoDzcd1vh5q5VfxLM4Afifqj6+aaCklZIGSBooaYUx5nZPBgOAs7njklZ68vau+s+ODI2ctULZBcWuIwFAtarqTRu/VfmMs1HW2pGSkiX9znOxAODbDUxqpheH9NTarzI1bMYKncgtch0JAKpNVctZkLX2aKXPj5/DsQBQ7W7o2kgzRiZp+5EcDZqWqqPZBa4jAUC1qGrB+tAY85ExZrQxZrSkf0h633OxAOD7XdG+vubckayDmfkaMC1V+0/kuY4EABesqgsCHpA0XVLXio/p1tpfeTIYAFRFn9b1tGBcijLzijVgaqp2Hj3lOhIAXJAqX5q01r5prb2/4uNtT4YCgHPRo3kdLZrQWyVlVoOmpWrzwWzXkQDgvH1nOTPG5Bhjss/ykWOM4bcfAK/RoVGsFt/ZW+EhQRo8PVWr9510HQkAzst3ljNrbYy1NvYsHzHW2tiaCgkAVXFRQrRev6uv6kaFacTMFVq285jrSABwzlhxCcCvNImrpcUT+6hZnUiNnrNKSzYfcR0JAM4J5QyA36kfE6HX7uytDg1jNHHBar23/qDrSABQZZQzAH4pLjJMC8f3VmKLOvrporVatPIr15EAoEooZwD8VnR4iObckawftk3Qg299qZe/2O06EgB8L8oZAL9WKyxY00ck6fouDfWnf2zR80t2yFrrOhYAfKsQ1wEAwNPCQoL0wuAeigz7Us8u2a5ThcX6zfUdZIxxHQ0A/gflDEBACAkO0pP9uyo6PEQzvtijU4Wl+tMtnRUcREED4F0oZwACRlCQ0e9v6qio8GBN+nSX8opK9NSAbgoN5g4PAN6DcgYgoBhj9MC17RUdHqq/fLhVuYWlemloD0WEBruOBgCSWBAAIEDddXlr/bFfJy3ZckRj565SbmGJ60gAIIlyBiCAjejTUk8P6KbUXcc1ctZKZeUXu44EAJQzAIGtf2JTTR7WUxvSMzVk+nIdP1XoOhKAAEc5AxDwruvcSC+P6qXdx05p4LRUHcrKdx0JQACjnAGApB+2TdC8MSk6kl2oAVNT9dXxPNeRAAQoyhkAVEhuVVevjE/RqcIS3T51mXYcyXEdCUAAopwBQCVdm8bptQl9ZCUNnJaqjQeyXEcCEGAoZwBwhnYNY/T6nX0UGRaiIdOXa9XeE64jAQgglDMAOIuW8VF6fWIfJcSEa8TMFfpiR4brSAACBOUMAL5F47haWjyxj1rFR2vsnDR9tOmw60gAAgDlDAC+Q3x0uBaN761OTWJ198I1enttuutIAPwc5QwAvkftyFAtGJuilFZ1df/i9VqwfJ/rSAD8GOUMAKogKjxEs0b30o/a1ddD72zUtM93uY4EwE9RzgCgiiJCgzV1RKJu7NpIf/5gq57+5zZZa13HAuBnQlwHAABfEhocpOcH91BUWIhe/GSnThWW6Hc3dFRQkHEdDYCfoJwBwDkKDjJ6on+X8kudS/cot7BEf76tq4IpaACqAeUMAM6DMUa/u7GDoiNC9MK/dii3qFTPDuyusBDuFgFwYShnAHCejDG6/+q2ig4P1uPvb1V+UakmD+upiNBg19EA+DD+xAOACzThstZ6/NYu+nTbUY2evVKnCktcRwLgwyhnAFANhqY013ODumvV3pMa9vIKZeYVuY4EwEdRzgCgmvTr3kRThvXUloPZGjx9uTJyCl1HAuCDKGcAUI2u6dRQs0b30r7jeRo4LVUHMvNdRwLgYyhnAFDNLr04XgvGJevYqUINnJqqPcdyXUcC4EMoZwDgAYkt6urV8b2VX1yqAVNTtfVwtutIAHwE5QwAPKRzk9pafGdvBQdJg6cv1/r9ma4jAfABlDMA8KA29WP0+p19FRMRoqEzlmv57uOuIwHwcpQzAPCw5vUi9fqdfdUorpZGzVqpT7cddR0JgBejnAFADWhYO0KvTeitNvWjNWFemt7/8pDrSAC8FOUMAGpIvehwvTK+t7o1jdO9r6zRG6vTXUcC4IUoZwBQg2rXCtW8scnq2zpe//f6es1dttd1JABehnIGADUsMixEL49K0tUdG+j3723SpE93uo4EwIt4tJwZY64zxmwzxuw0xjx4lu3NjTGfGmPWGmM2GGOur7Tt1xXHbTPGXOvJnABQ0yJCgzV5WE/d0r2x/vrRNv3lw62y1rqOBcALhHjqCxtjgiVNknS1pHRJq4wx71lrN1fa7SFJi621U4wxHSW9L6llxevBkjpJaixpiTGmrbW21FN5AaCmhQYH6ZmB3RUZHqIpn+1SbmGJ/nBTJwUFGdfRADjksXImKVnSTmvtbkkyxiyS1E9S5XJmJcVWvK4t6WDF636SFllrCyXtMcbsrPh6qR7MCwA1LijI6LFbOis6PETT/71bpwpL9GT/rgoJ5q4TIFB5spw1kbS/0ufpklLO2OcPkv5pjPmJpChJV1U6dvkZxzY58xsYYyZImiBJzZs3r5bQAFDTjDH69Y/bKzo8RM98vF05BSV6cUgPRYQGu44GwAHXf5oNkTTHWttU0vWS5htjqpzJWjvdWptkrU1KSEjwWEgA8DRjjO678mI9cnMnLdlyRCNnrVR2QbHrWAAc8GQ5OyCpWaXPm1a8V9lYSYslyVqbKilCUnwVjwUAvzOqb0s9P7iH1n51UoOmLdfR7ALXkQDUME+Ws1WSLjbGtDLGhKn8Bv/3ztjnK0lXSpIxpoPKy1lGxX6DjTHhxphWki6WtNKDWQHAa9zcrbFmjuqlfcdzdfvUVO07nus6EoAa5LFyZq0tkXSvpI8kbVH5qsxNxphHjTE3V+z2C0njjTHrJb0qabQtt0nlZ9Q2S/pQ0j2s1AQQSC5rm6CF41KUU1Cs/lNStfFAlutIAGqI8Ze5OklJSTYtLc11DACoVjuPntLImSuUU1Ci6SOT1Kd1PdeRAFQDY8xqa23S2ba5XhAAAPgObepH6827+6pB7QiNmr1SH2487DoSAA+jnAGAl2tUu5Zev7OPOjWO1d0LV2vRyq9cRwLgQZQzAPABdaLCtHBcin5wcYIefOtLTfp0J497AvwU5QwAfMTpB6affh7no3/frLIyChrgbzz5hAAAQDU7/TzOulHhmrV0j07kFumvt3dTWAh/awP+gnIGAD4mKMjodzd2UHxMmJ78cJsy84o1ZXhPRYbxKx3wB/ypBQA+yBijuy9voydu66IvdmRo6IwVOplb5DoWgGpAOQMAHzY4ubmmDE/U5kPZGjAtVQcz811HAnCBKGcA4OOu7dRQ88Yk60hWgW6fskw7j55yHQnABaCcAYAf6H1RPS26s7eKSq0GTF2mtV+ddB0JwHminAGAn+jUuLbevKuPYiJCNXTGCn2+PcN1JADngXIGAH6kRb0ovXFXH7WMj9K4uav07roDriMBOEeUMwDwM/VjIvTanb3Vo3kd/ey1dZqzdI/rSADOAeUMAPxQbESo5o1J1tUdGugPf9usZ/65jcc9AT6CcgYAfioiNFiTh/XUoKRmeuGTnfrtOxtVyuOeAK/HOGkA8GMhwUF6on8X1YsO0+TPdulkbpGeHdRdEaHBrqMB+BaUMwDwc8YY/fK69qoXHa4//n2zMvNWafrIRMVEhLqOBuAsuKwJAAFi7KWt9Nyg7lq194QGT1+ujJxC15EAnAXlDAACyC09mmjGqCTtyjilAVOXaf+JPNeRAJyBcgYAAeaKdvW1cFxvncwr1m1TlmnLoWzXkQBUQjkDgACU2KKOXp/YR8HGaOC0VK3cc8J1JAAVKGcAEKDaNojRm3f3VUJMuEbMXKGPNx9xHQmAKGcAENCaxNXSGxP7qn3DGE1csFqL0/a7jgQEPMoZAAS4ulFhemV8b/VtXU+/fGODpn6+y3UkIKBRzgAAigoP0cxRvXRj10Z64oOtevz9LSrjaQKAEwyhBQBIksJCgvTC4B6qFxWm6f/ereOnivRE/y4KDebveKAmUc4AAF8LCjL6w82dFB8drqc/3q7MvCK9NLSnaoXxuCegpvDnEADgG4wx+smVF+uxWzvrk21HNWLmCmXlFbuOBQQMyhkA4KyGpbTQpKE9tSE9SwOnpepwVoHrSEBAoJwBAL7V9V0aac4dvZR+Mk/9pyzT7oxTriMBfo9yBgD4Tn3bxGvRhD4qKC7V7VNTtSE903UkwK9RzgAA36tL09p6466+igwL1pDpy7V05zHXkQC/RTkDAFRJq/govXlXXzWrG6k7Zq/SPzYcch0J8EuUMwBAlTWIjdBrd/ZRt2a1de+razR/+T7XkQC/QzkDAJyT2rVCNX9siq5sX1+/e2ejnluyXdbyNAGgulDOAADnLCI0WFOHJ+r2xKZ6bskOPfzuJpXyuCegWvCEAADAeQkJDtJfb++qetFhmvb5bp3IK9IzA7spPISnCQAXgnIGADhvxhj9+scdVC8qTI+/v1VZecWaOiJR0eH87wU4X1zWBABcsAmXtdZTA7opdfdxDZuxXMdPFbqOBPgsyhkAoFrcnthU00ckauvhHA2Ylqr0k3muIwE+iXIGAKg2V3ZooAXjUnQsp1C3T0nV9iM5riMBPodyBgCoVr1a1tXiiX1kZTVgaqpW7zvhOhLgUyhnAIBq175hrN6Y2Fd1o8I07OUV+nTrUdeRAJ9BOQMAeESzupF6fWIfXVw/RuPmpemtNemuIwE+gXIGAPCY+OhwvTqht3pfVFf3L16vl7/Y7ToS4PUoZwAAj4oOD9Gs0b10fZeG+tM/tujP729RGU8TAL4VUwIBAB4XHhKsF4f0VL2oTZr2793afzJPzwzsrohQniYAnIkzZwCAGhEcZPRov0566IYO+mDjYQ2avlwZOQyrBc5EOQMA1BhjjMb94CJNG56o7YdzdMukpcxCA85AOQMA1LhrOjXU4jv7qLi0TP0nL9O/t2e4jgR4DcoZAMCJLk1r6517LlGTOrV0x5xVWrhin+tIgFegnAEAnGkcV0tv3NVXl10cr9++vVGP/WMzKzkR8ChnAACnosNDNGNkkkb3bakZX+zRxAWrlVdU4joW4AzlDADgXEhwkP5wcyf94aaOWrLliAZNW66j2QWuYwFOUM4AAF5j9CWt9PKoJO3KOKVbJi3VlkPZriMBNY5yBgDwKj9q30CvT+yjMivdPmWZPt3GQ9MRWChnAACv06lx+UrOlvFRGjtnleal7nUdCagxlDMAgFdqWDtCi+/sox+1r6+H392kR/62SaWs5EQAoJwBALxWVHiIpo1I0thLW2n20r26c36acgtZyQn/RjkDAHi14CCj393YUX+8pbM+2XpUA6el6nAWKznhvyhnAACfMKJ3C80a3Uv7jufplklLtelglutIgEdQzgAAPuPydvX1xl19FGSkAVNT9a8tR1xHAqod5QwA4FPaN4zVO/dcojb1ozV+XppmL93jOhJQrShnAACfUz82Qq9N6KOrOzbQI3/brN+/u1ElpWWuYwHVgnIGAPBJtcKCNWVYoiZcdpHmpu7TuHlpOsVKTvgByhkAwGcFBRn95voOevzWLvpixzHdPmWZDmbmu44FXBDKGQDA5w1Naa45d/TSgZP5umXSUn2ZzkpO+C7KGQDAL/zg4gS9eXdfhQYHaeC0VH206bDrSMB5oZwBAPxG2wYxeueeS9SuYYwmLlitGf/eLWt55BN8C+UMAOBXEmLCtWhCb13fuZEee3+LfvvORhWzkhM+JMR1AAAAqltEaLBeHNJDLepFavJnu7T/RJ4mDeup2IhQ19GA78WZMwCAXwoKMvrlde31ZP+uSt11XLdPWab0k3muYwHfi3IGAPBrA3s107wxyTqcVaBbJi3Tuv2ZriMB34lyBgDwe33bxOutuy9RrbAgDZqWqg++POQ6EvCtKGcAgIDQpn603rn7EnVuUlt3LVyjqZ/vYiUnvBLlDAAQMOpFh2vhuBTd1K2xnvhgqx5880tWcsLrsFoTABBQIkKD9fyg7mpVL1IvfLJT6Zl5mjwsUbVrsZIT3oEzZwCAgBMUZHT/Ne309IBuWrnnhPpPWab9J1jJCe9AOQMABKz+iU01f2yKMnIKdcukpVq976TrSADlDAAQ2HpfVE9v391XMREhGjJjuf62/qDrSAhwlDMAQMC7KCFab919ibo3jdNPXl2rSZ/uZCUnnKGcAQAgqW5UmOaPS9atPZrorx9t0wNvbFBRCSs5UfNYrQkAQIXwkGA9M7CbWtSL1HNLdij9ZJ6mDk9UXGSY62gIIJw5AwCgEmOMfnZVWz03qLvW7MvUbZOXae+xXNexEEAoZwAAnMUtPZpo4fgUncwr0q2Tl2rV3hOuIyFAUM4AAPgWvVrW1dt3X6I6kWEaNmOF3l13wHUkBADKGQAA36FlfJTeuruveraI008XrdNzS7azkhMeRTkDAOB7xEWGad6YFPXv2VTPLdmh+xevV2FJqetY8FOs1gQAoArCQoL01ICuuighSn/9aJsOnMzXtBGJqhPFSk5UL86cAQBQRcYY3XNFG704pIfWpWfq1slLtTvjlOtY8DOUMwAAztFN3Rrr1fG9lVNQotumLNPy3cddR4IfoZwBAHAeElvU0dt3X6J6UWEaMXOF3lyd7joS/ATlDACA89S8XqTeuvsSJbeqq1+8vl7P/HMbKzlxwShnAABcgNq1QjXnjmQNSmqmFz7ZqfsWrVNBMSs5cf5YrQkAwAUKDQ7SE/27qGV8lJ78aKu+Op6raSOS1LB2hOto8EGcOQMAoBoYY3TX5a01bXiidh49pZtf+o/W7c90HQs+iHIGAEA1uqZTQ715d1+FhwZp4LRUvbOWRz7h3FDOAACoZu0bxurdey5Vz+Zx+tlr6/TEB1tVWsZCAVQN5QwAAA+oGxWm+WNTNCyluaZ+vksT5qUpp6DYdSz4AMoZAAAeEhocpMdu7aI/3tJZn23P0G2Tl2nf8VzXseDlKGcAAHjYiN4tNH9ssjJOFarfpKVatvOY60jwYpQzAABqQN/W8Xr3nkuUEB2uEbNWal7qXgbW4qwoZwAA1JAW9aL01t19dUW7BD387ib99p2NKiopcx0LXoZyBgBADYqJCNW0EUm66/LWemXFVxoxc4VO5Ba5jgUv4tFyZoy5zhizzRiz0xjz4Fm2P2uMWVfxsd0Yk1lpW2mlbe95MicAADUpOMjoV9e11/ODu2vd/kzd/NJ/tPVwtutY8BIeK2fGmGBJkyT9WFJHSUOMMR0r72Ot/bm1tru1trukFyW9VWlz/ult1tqbPZUTAABX+nVvosV39lFxaZn6T16mf2467DoSvIAnz5wlS9pprd1trS2StEhSv+/Yf4ikVz2YBwAAr9OtWZzeu/dStWkQownzV+ulT3awUCDAebKcNZG0v9Ln6RXv/Q9jTAtJrSR9UuntCGNMmjFmuTHmlm85bkLFPmkZGRnVlRsAgBrVIDZCr03orVu6N9ZT/9yun7y6VvlFpa5jwRFvWRAwWNIb1trK/yW2sNYmSRoq6TljTOszD7LWTrfWJllrkxISEmoqKwAA1S4iNFjPDuquB3/cXv/48pAGTkvVoax817HggCfL2QFJzSp93rTivbMZrDMuaVprD1T8c7ekzyT1qP6IAAB4D2OMJv6wtV4emaQ9x3J104tLteark65joYZ5spytknSxMaaVMSZM5QXsf1ZdGmPaS6ojKbXSe3WMMeEVr+MlXSJpswezAgDgNa7s0EBv3d1XkWHBGjxtud5cne46EmqQx8qZtbZE0r2SPpK0RdJia+0mY8yjxpjKqy8HS1pkv3n3YwdJacaY9ZI+lfSEtZZyBgAIGG0bxOjdey5RYos6+sXr6/X4+1tUWsZCgUBg/GVFSFJSkk1LS3MdAwCAalVcWqY//X2z5qbu0+XtEvTCkB6KjQh1HQsXyBizuuLe+v/hLQsCAADAWYQGB+mRfp31+K1d9J8dx3TrpKXacyzXdSx4EOUMAAAfMDSluRaMS9GJ3CL1e+k/+mIHI6T8FeUMAAAf0fuienrv3kvVOK6WRs9epdlL9zCw1g9RzgAA8CHN6kbqjbv66kft6+uRv23Wg29+qaKSMtexUI0oZwAA+Jjo8BBNG56oe69oo9fS9mvYy8t17FSh61ioJpQzAAB8UFCQ0f9d204vDumhDelZ6vfSUm0+mO06FqoB5QwAAB92U7fGemNiX5WWWfWfskwfbjzkOhIuEOUMAAAf16Vpbb137yVq3yhGExes0fNLdrBQwIdRzgAA8AP1YyP06vjeuq1nEz27ZLvufWWt8opKXMfCeaCcAQDgJyJCg/X0gG767fUd9MHGQ7p9SqoOZOa7joVzRDkDAMCPGGM0/rKLNHN0L+0/kad+/9/evQdXWd95HP98c4MEApIWLE0oiSAgREHQGJJs1bKtOuWmBWsFasMWRjah2tnxOu2s1U7b2W3p1hgECwptAely2Whw2iprqQmGiwihgAQbNCDh0igxIASS/PaPnHbojrY5geT3nHPer7/Ik5B8Ms8Q3jnnSZ6nKrT9nfd9z0IYiDMAAKLQzcMHaH1Rnnr3SNDXfl6lX28/5HsSOog4AwAgSg0dkKqyogLdkPUpPbimWk+U71VLK7+wNuiIMwAAoljflEQtK7xe38jL1NKKg5q9fLsaz5z3PQt/B3EGAECUS4iP02OTR+lHd1yt1//0Z91eWqk/nTjlexY+AXEGAECMuCvnc1rxzVw1njmvqaWV2lRzwvckfAziDACAGJKTlaay4nxl9EtR4XNbteS1Wn5hbcAQZwAAxJiMfilac+94fWnkZ/T9Dfv0wJpqNbe0+p6FEOIMAIAY1KtHghbOGKv7JlypNW8c1t0/36ITTc2+Z0HEGQAAMSsuzvTtLw5T6d1jtedIoyY/VaE/vtfoe1bMI84AAIhxX75moNbcmyeTNG3RZm2orvc9KaYRZwAAQNnpfVVWXKBRn+2ropU7tODlGrW18YMCPhBnAABAktQ/tYdWzrlB08dl6MmNBzRvxRs63dzie1bMIc4AAMBf9UiI139Mu0bfnThSL+89pq88vVmH3v/I96yYQpwBAIC/YWb6l4IsPVeYo/dOntGU0kptPfi+71kxgzgDAAAf68Zh/VVWlK/LkhM1Y0mVnt9a53tSTCDOAADAJ7qif2+tL8rX+CGf1sPrduuxF/aopbXN96yoRpwBAIC/q29yop695zp9syBLyza/o1lLt6rhFL+wtqsQZwAA4B9KiI/TdyaO1E+mj9aOug80qaRCuw6d9D0rKhFnAACgw74yLkNr5+XJzDR90etavY3r0C414gwAAIQlO72vyucX6IYr0vTQ2t16ZN1ubpx+CRFnAAAgbP16JWlZYY7+9aYhWrW1TncurlJ94xnfs6ICcQYAADolPs704K0jtGjmWL19rEmTSipUVdvge1bEI84AAMBFuTV7oMqK89UnOVEzlmzRktdq5Rz35ews4gwAAFy0oQNSVVaUrwkjBuj7G/bpvud36qNz3JezM4gzAABwSaT2TNSimeP0wC3D9WL1Ed2xcLPebTjte1bEIc4AAMAlExdnKrp5qJYX5ujoh2c1qaRCr7513PesiEKcAQCAS+7zw/rrxeICZfRL0ezl2/SzVw6orY3r0DqCOAMAAF1iUFqK1s7L0+1j0vXTV2o05xfb1XjmvO9ZgUecAQCALpOcFK+f3Dla35s8SptqTmhqaaX2H23yPSvQiDMAANClzEz35GVq1dxcnWpudPUJCAAACW9JREFU0e0LK1VefcT3rMAizgAAQLe4PjNN5fMLdNXAPipe+aZ+8NI+tbS2+Z4VOMQZAADoNpf36alVc3L19fGD9cwfajVr6VY1nGr2PStQiDMAANCtkhLi9PiUbP14+mjtqPtAk0oqtOvQSd+zAoM4AwAAXkwbl6G18/JkZpq+6HWt3lbne1IgEGcAAMCb7PS+Kp9foBuuSNNDa3fr0fW71dzS6nuWV8QZAADwql+vJC0rzNG8m4Zo5ZY6fXVxleobz/ie5Q1xBgAAvIuPMz106wg9PWOsDhxr0qSSClXVNvie5QVxBgAAAuO2qweqrDhffZITNWPJFi2tOCjnYuu2T8QZAAAIlKEDUlVWlK8JIwboifK9un/1Tn10rsX3rG5DnAEAgMBJ7ZmoRTPH6YFbhuuFXUd0x8LNerfhtO9Z3YI4AwAAgRQXZyq6eaiWFeaovvGsJpVU6NX9x33P6nLEGQAACLQbh/VX+fwCZfRL0exl2/TkxgNqa4ve69CIMwAAEHiD0lK0dl6epo5J14KXazT3l9v14dnzvmd1CeIMAABEhOSkeC24c7S+N3mUfr//hKY8VamaY02+Z11yxBkAAIgYZqZ78jK1am6uTjW3aGpppTZU1/uedUkRZwAAIOJcn5mm8vkFumpgHxWt3KEfvrRPLa1tvmddEsQZAACISJf36alVc3I1K3ewFv+hVl9/dqsaTjX7nnXRiDMAABCxkhLi9MTUbP3ntGu0/d0PNKmkQtWHT/qedVGIMwAAEPGmXzdI6+blycw0bdHr+vW2Q74ndRpxBgAAokJ2el+9OL9AOZlpenBttR5dv1vNLa2+Z4WNOAMAAFEjrVeSls/O0b03DtHKLXW665kqHW0863tWWIgzAAAQVeLjTA/fNkJPzxirmqNNmljymqpqG3zP6jDiDAAARKXbrh6o/ynKV5+eiZqxZIuerTgo54J/2yfiDAAARK0rL09VWXG+vjBigB4v36v7V+/UmXPBvg6NOAMAAFEttWeiFs8cpwduGa4Xdh3R7QsrVdfwke9Zn4g4AwAAUS8uzlR081AtK8xRfeNZTSx5Ta/uP+571scizgAAQMy4cVh/vVhcoIx+KZq9bJue3HhAbW3Bug6NOAMAADHlc59K0dp5eZo6Jl0LXq7R3F++oQ/Pnvc966+IMwAAEHOSk+K14M7RemzSSP1+/3FNeapSNceafM+SRJwBAIAYZWb6Rn6WVs7JVdPZFk0trdSG6nrfs4gzAAAQ23Ky0rThWwUa8ZlUFa3coR++tE8trW3e9hBnAAAg5l3ep6eenztes3IHa/+xJpmZty0J3j4yAABAgCQlxOmJqdk619Km+Dh/ccYjZwAAABdISvCbR8QZAABAgBBnAAAAAUKcAQAABAhxBgAAECDEGQAAQIAQZwAAAAFCnAEAAAQIcQYAABAgxBkAAECAEGcAAAABQpwBAAAECHEGAAAQIMQZAABAgBBnAAAAAUKcAQAABAhxBgAAECDEGQAAQIAQZwAAAAFCnAEAAAQIcQYAABAgxBkAAECAEGcAAAABYs453xsuCTM7IendbvhQn5b05274OOg6nMPIxvmLfJzDyMc5vHiDnXP9P+4VURNn3cXMtjvnrvO9A53HOYxsnL/IxzmMfJzDrsXTmgAAAAFCnAEAAAQIcRa+Z3wPwEXjHEY2zl/k4xxGPs5hF+KaMwAAgADhkTMAAIAAIc4AAAAChDjrIDO71cz2m9nbZvaw7z0Ij5kNMrNXzWyvme0xs/t8b0LnmFm8mb1pZuW+tyB8ZnaZma0xs7fMbJ+Zjfe9CR1nZt8OfQ39o5mtMrOevjdFI+KsA8wsXlKppNskjZT0NTMb6XcVwtQi6d+ccyMl5Uoq4hxGrPsk7fM9Ap32M0m/cc6NkDRanMuIYWbpkr4l6TrnXLakeEl3+V0VnYizjsmR9LZzrtY5d07S85KmeN6EMDjn6p1zO0J/blL7fwjpflchXGaWIenLkpb43oLwmVlfSZ+XtFSSnHPnnHMn/a5CmBIkJZtZgqQUSUc874lKxFnHpEs6dMHLh8V/7BHLzDIlXStpi98l6IT/kvSgpDbfQ9ApWZJOSHou9NT0EjPr5XsUOsY5956kH0uqk1QvqdE59zu/q6ITcYaYYma9Ja2VdL9z7kPfe9BxZjZR0nHn3Bu+t6DTEiSNlfS0c+5aSaclcQ1vhDCzfmp/1ihL0mcl9TKzmX5XRSfirGPekzTogpczQscQQcwsUe1htsI5t873HoQtX9JkM3tH7ZcWfMHMfuV3EsJ0WNJh59xfHrVeo/ZYQ2T4Z0kHnXMnnHPnJa2TlOd5U1Qizjpmm6QrzSzLzJLUfgHkC543IQxmZmq/zmWfc26B7z0In3PuEedchnMuU+3/Bv/XOcd37RHEOXdU0iEzGx46NEHSXo+TEJ46SblmlhL6mjpB/EBHl0jwPSASOOdazKxY0m/V/tMpzzrn9niehfDkS5olabeZ7Qwde9Q595LHTUAsmi9pRegb3VpJhZ73oIOcc1vMbI2kHWr/Cfg3xW2cugS3bwIAAAgQntYEAAAIEOIMAAAgQIgzAACAACHOAAAAAoQ4AwAACBDiDAAukpndZGblvncAiA7EGQAAQIAQZwBihpnNNLOtZrbTzBabWbyZnTKzn5rZHjPbaGb9Q287xsyqzKzazNaH7isoMxtqZq+Y2S4z22FmQ0LvvreZrTGzt8xsReg3qANA2IgzADHBzK6S9FVJ+c65MZJaJc2Q1EvSdufcKEmbJP176K/8QtJDzrlrJO2+4PgKSaXOudFqv69gfej4tZLulzRS0hVqvysFAISN2zcBiBUTJI2TtC30oFaypOOS2iStDr3NryStM7O+ki5zzm0KHV8u6b/NLFVSunNuvSQ5585KUuj9bXXOHQ69vFNSpqSKrv+0AEQb4gxArDBJy51zj/zNQbPv/r+36+w97Zov+HOr+PoKoJN4WhNArNgoaZqZDZAkM0szs8Fq/zo4LfQ2d0uqcM41SvrAzP4pdHyWpE3OuSZJh81sauh99DCzlG79LABEPb6zAxATnHN7zew7kn5nZnGSzksqknRaUk7odcfVfl2aJN0jaVEovmolFYaOz5K02MweD72P6d34aQCIAeZcZx/BB4DIZ2annHO9fe8AgL/gaU0AAIAA4ZEzAACAAOGRMwAAgAAhzgAAAAKEOAMAAAgQ4gwAACBAiDMAAIAA+T+skNvK66t78gAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# dot_img_file = \"d_mod-\"+str(d_model)+\"-d_k-\"+str(d_k)+\"-d_v-\"+str(d_v)+\"-n_heads-\"+str(n_heads)+\"-enc_lays-\"+str(n_encoder_layers)+\"-dec_lays-\"+str(n_decoder_layers)+'-transformer.png'\n",
        "# tf.keras.utils.plot_model(transformer, to_file=dot_img_file, show_shapes=True)"
      ],
      "metadata": {
        "id": "RSK1eufbeQPL"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#transformer = keras.models.load_model('./Models/')"
      ],
      "metadata": {
        "id": "Oly3kYeU3FnV"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Testing"
      ],
      "metadata": {
        "id": "KJ_Gx7RVeNla"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def greedy_translate(model, source_sentence, target_sentence_start=sp_ben_bpe.bos_id()):\n",
        "  # Tokenizing and padding\n",
        "  source_seq = tokenize(source_sentence, sp_en_bpe)\n",
        "  source_seq = tf.keras.preprocessing.sequence.pad_sequences(source_seq, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "  predict_seq = [[target_sentence_start]]\n",
        "  \n",
        "  predict_sentence = [target_sentence_start] # Deep copy here to prevent updates on target_sentence_start\n",
        "  while predict_sentence[-1] != sp_ben_bpe.eos_id() and len(predict_sentence) < MAX_TOKENS:\n",
        "    predict_output = model([np.array(source_seq), np.array(predict_seq)], training=None)\n",
        "    predict_label = tf.argmax(predict_output, axis=-1) # Pick the label with highest softmax score\n",
        "    predict_seq = tf.concat([predict_seq, predict_label], axis=-1) # Updating the prediction sequence\n",
        "    predict_sentence.append(predict_label[0][0].numpy())\n",
        "  \n",
        "  if len(predict_sentence) == MAX_TOKENS:\n",
        "      return detokenize(predict_sentence, sp_ben_bpe, True, False)\n",
        "  return detokenize(predict_sentence, sp_ben_bpe)"
      ],
      "metadata": {
        "id": "zfg1AaKDnPQM"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow_probability as tfp"
      ],
      "metadata": {
        "id": "pLrtBO__iPPl"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", greedy_translate(transformer, sentence))"
      ],
      "metadata": {
        "id": "KtWeJl1JoapJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e388b8d3-1702-4d99-e5ec-b1e5f01d4c17"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I love you.\n",
            "Bengali Sentence:  আমি তোমাকে ভালবাসি।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I am tired.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", greedy_translate(transformer, sentence))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K2axdVLjq6z1",
        "outputId": "2b0e9863-5268-4521-df2d-0022b21a7ba8"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I am tired.\n",
            "Bengali Sentence:  আমি ক্লান্ত হয়ে গেছি।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = list(test_val_data['src'][:250])\n",
        "refs = [list(test_val_data['tgt'][:250])]\n",
        "sys = [greedy_translate(transformer, sentence) for sentence in sentences]"
      ],
      "metadata": {
        "id": "9XNckV-wBgiN"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bleu = BLEU()\n",
        "bleu.corpus_score(sys, refs)"
      ],
      "metadata": {
        "id": "otRhPJC1VKyj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34eef648-c50a-4ace-9600-ea8391e2be93"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BLEU = 0.99 14.9/2.4/0.4/0.1 (BP = 0.822 ratio = 0.836 hyp_len = 2995 ref_len = 3583)"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chrf = CHRF()\n",
        "chrf.corpus_score(sys, refs)"
      ],
      "metadata": {
        "id": "mlhToSrvYNqd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "326f55c5-f5cf-449f-abca-61225a4adb93"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "chrF2 = 20.48"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ter = TER()\n",
        "ter.corpus_score(sys, refs)"
      ],
      "metadata": {
        "id": "DtIadLMJYe1w",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1015997-608c-4bb2-8c35-61e7fa13250d"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TER = 98.69"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dev_perf = pd.DataFrame({\"src\":sentences, \"sys\":sys, \"refs\":refs[0]})\n",
        "train_dev_perf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "ftWTPJzHc2FA",
        "outputId": "9d3d03af-cc1b-4161-d944-0299ee78752b"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   src  \\\n",
              "0    Through her work, she spread the message of th...   \n",
              "1    Interacting with beneficiaries and store owner...   \n",
              "2                                       These include:   \n",
              "3    It is no surprise that today Japan is India’s ...   \n",
              "4           Record growth in last two and a half years   \n",
              "..                                                 ...   \n",
              "245  •India will emerge as a global leader in trans...   \n",
              "246        BHIM stands for Bharat Interface for Money.   \n",
              "247          Heartiest congratulations, Beti Shivangi.   \n",
              "248                But, we are now awaiting the rains.   \n",
              "249  As reported by the Swiss Bank Authorities, the...   \n",
              "\n",
              "                                                   sys  \\\n",
              "0    তার কাজের মাধ্যমে তিনি নিরাপদ এবং নিরাপদ মানুষ...   \n",
              "1    কেন্দ্রীয় সরকার গত ২৪ ঘন্টায় বিভিন্ন ধরনের স...   \n",
              "2                                     এর মধ্যে রয়েছে:   \n",
              "3    জাপানের বৃহত্তম বিনিয়োগ, যার মধ্যে ভারতেই নয়...   \n",
              "4                                     দুই বছরে দ্বিগুণ   \n",
              "..                                                 ...   \n",
              "245  ভারত বনাম ব্রিটেন এই নিয়ে আলোচনা করবে, যেখানে...   \n",
              "246                          ভারতের জন্য সুখবর রয়েছে।   \n",
              "247                    পরম প্রশংসার্হ, ডাল, বিড়ি ভোট।   \n",
              "248                       তবে, আমরা চিরকাল বৃষ্টি করি।   \n",
              "249  বাসাঘর জানিয়েছে, অবৈধভাবে কালোদের বিরুদ্ধে কা...   \n",
              "\n",
              "                                                  refs  \n",
              "0    নিজের কর্মেরউদাহরণ স্থাপন করে তিনি মানুষকে সেব...  \n",
              "1    সারা দেশের ৫ হাজারেরও বেশি স্থান থেকে দোকান মা...  \n",
              "2                         এই প্রকল্পগুলির মধ্যে রয়েছে-  \n",
              "3    জাপান যে বর্তমানে ভারতের চতুর্থ বৃহত্তম প্রত্য...  \n",
              "4                    গত আড়াই বছরেরেকর্ড পরিমাণ অগ্রগতি  \n",
              "..                                                 ...  \n",
              "245  · সর্বাধুনিক প্রযুক্তি এবং দক্ষ শ্রমিকের ব্যবহ...  \n",
              "246                  BHIM মানে ভারত ইন্টারফেস ফর মানি।  \n",
              "247               কন্যা শিবাঙ্গীকে অনেক অনেক অভিনন্দন!  \n",
              "248                           বর্ষার প্রতীক্ষা চলছেই ।  \n",
              "249  স্যুইস ব্যাঙ্কবলেছে যে ভারতীয়দের দ্বারা জমা ধন...  \n",
              "\n",
              "[250 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7203d8cc-842f-4cb3-8301-c926529c6cb3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>src</th>\n",
              "      <th>sys</th>\n",
              "      <th>refs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Through her work, she spread the message of th...</td>\n",
              "      <td>তার কাজের মাধ্যমে তিনি নিরাপদ এবং নিরাপদ মানুষ...</td>\n",
              "      <td>নিজের কর্মেরউদাহরণ স্থাপন করে তিনি মানুষকে সেব...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Interacting with beneficiaries and store owner...</td>\n",
              "      <td>কেন্দ্রীয় সরকার গত ২৪ ঘন্টায় বিভিন্ন ধরনের স...</td>\n",
              "      <td>সারা দেশের ৫ হাজারেরও বেশি স্থান থেকে দোকান মা...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>These include:</td>\n",
              "      <td>এর মধ্যে রয়েছে:</td>\n",
              "      <td>এই প্রকল্পগুলির মধ্যে রয়েছে-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>It is no surprise that today Japan is India’s ...</td>\n",
              "      <td>জাপানের বৃহত্তম বিনিয়োগ, যার মধ্যে ভারতেই নয়...</td>\n",
              "      <td>জাপান যে বর্তমানে ভারতের চতুর্থ বৃহত্তম প্রত্য...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Record growth in last two and a half years</td>\n",
              "      <td>দুই বছরে দ্বিগুণ</td>\n",
              "      <td>গত আড়াই বছরেরেকর্ড পরিমাণ অগ্রগতি</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>245</th>\n",
              "      <td>•India will emerge as a global leader in trans...</td>\n",
              "      <td>ভারত বনাম ব্রিটেন এই নিয়ে আলোচনা করবে, যেখানে...</td>\n",
              "      <td>· সর্বাধুনিক প্রযুক্তি এবং দক্ষ শ্রমিকের ব্যবহ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>246</th>\n",
              "      <td>BHIM stands for Bharat Interface for Money.</td>\n",
              "      <td>ভারতের জন্য সুখবর রয়েছে।</td>\n",
              "      <td>BHIM মানে ভারত ইন্টারফেস ফর মানি।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>247</th>\n",
              "      <td>Heartiest congratulations, Beti Shivangi.</td>\n",
              "      <td>পরম প্রশংসার্হ, ডাল, বিড়ি ভোট।</td>\n",
              "      <td>কন্যা শিবাঙ্গীকে অনেক অনেক অভিনন্দন!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>248</th>\n",
              "      <td>But, we are now awaiting the rains.</td>\n",
              "      <td>তবে, আমরা চিরকাল বৃষ্টি করি।</td>\n",
              "      <td>বর্ষার প্রতীক্ষা চলছেই ।</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>249</th>\n",
              "      <td>As reported by the Swiss Bank Authorities, the...</td>\n",
              "      <td>বাসাঘর জানিয়েছে, অবৈধভাবে কালোদের বিরুদ্ধে কা...</td>\n",
              "      <td>স্যুইস ব্যাঙ্কবলেছে যে ভারতীয়দের দ্বারা জমা ধন...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>250 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7203d8cc-842f-4cb3-8301-c926529c6cb3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7203d8cc-842f-4cb3-8301-c926529c6cb3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7203d8cc-842f-4cb3-8301-c926529c6cb3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def sampling_translate(model, source_sentence, temperature=0.6, target_sentence_start=sp_ben_bpe.bos_id()):\n",
        "  # Tokenizing and padding\n",
        "  source_seq = tokenize(source_sentence, sp_en_bpe)\n",
        "  source_seq = tf.keras.preprocessing.sequence.pad_sequences(source_seq, maxlen = MAX_TOKENS, padding='post', truncating='post')\n",
        "  predict_seq = [[target_sentence_start]]\n",
        "  logprob = 0\n",
        "  \n",
        "  predict_sentence = [target_sentence_start] # Deep copy here to prevent updates on target_sentence_start\n",
        "  while predict_sentence[-1] != sp_ben_bpe.eos_id() and len(predict_sentence) < MAX_TOKENS:\n",
        "    predict_output = model([np.array(source_seq), np.array(predict_seq)], training=None)\n",
        "    probs = predict_output[-1].numpy()[0]\n",
        "    log_probs = np.log(probs)\n",
        "    u = np.random.uniform(low=1e-6, high=1.0 - 1e-6, size=log_probs.shape)\n",
        "    g = -np.log(-np.log(u))\n",
        "    sample_idx = np.argmax(log_probs + g * temperature, axis=-1)\n",
        "    sample_prob = predict_output[-1].numpy()[0][sample_idx]\n",
        "    logprob += np.log(sample_prob)\n",
        "    predict_label = np.arange(predict_output.shape[-1])[sample_idx] # Sample the label softmax score\n",
        "    predict_seq = tf.concat([predict_seq, tf.cast([[predict_label]], dtype=tf.int32)], axis=-1) # Updating the prediction sequence\n",
        "    predict_sentence.append(predict_label)\n",
        "  \n",
        "  if len(predict_sentence) == MAX_TOKENS:\n",
        "      return detokenize(predict_sentence, sp_ben_bpe, True, False), logprob\n",
        "  return detokenize(predict_sentence, sp_ben_bpe), logprob"
      ],
      "metadata": {
        "id": "2dmVo_eQhE8v"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", sampling_translate(transformer, sentence))"
      ],
      "metadata": {
        "id": "CB5iHHlxh1cC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "af634d7b-c80e-4525-f5f0-8fb43279cb91"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I love you.\n",
            "Bengali Sentence:  ('আমি তোমাকে ভালবাসি।', -1.64567662589252)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I am hungry.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", sampling_translate(transformer, sentence))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IiZmzCY3q1S9",
        "outputId": "1b058b00-cf35-4eb4-98c6-7d64c2889aab"
      },
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I am hungry.\n",
            "Bengali Sentence:  ('আমি ক্লান্ত হয়ে পড়ি।', -3.066206020768732)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_samples(sentence, n_samples, model=None, temperature=0.6):\n",
        "    samples, log_probs = [], []\n",
        "    for _ in range(n_samples):\n",
        "        sample, logp = sampling_translate(model, sentence, temperature)\n",
        "        samples.append(sample)\n",
        "        log_probs.append(logp)\n",
        "    return samples, log_probs"
      ],
      "metadata": {
        "id": "xIxJ9HqTtxHY"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def rouge1_similarity(system, reference):\n",
        "    sys_counter = Counter(system)\n",
        "    ref_counter = Counter(reference)\n",
        "    overlap = 0\n",
        "    for token in sys_counter:\n",
        "        token_count_sys = sys_counter[token]\n",
        "        token_count_ref = ref_counter[token]\n",
        "        overlap += min(token_count_ref, token_count_sys)\n",
        "    precision = overlap / sum(sys_counter.values())\n",
        "    recall = overlap / sum(ref_counter.values())\n",
        "    if precision + recall != 0:\n",
        "        rouge1_score = 2 * ((precision * recall)/(precision + recall))\n",
        "    else:\n",
        "        rouge1_score = 0     \n",
        "    return rouge1_score"
      ],
      "metadata": {
        "id": "uhIlVoVwt7YC"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def weighted_avg_overlap(samples, log_probs):\n",
        "    scores = {}\n",
        "    for index_candidate, candidate in enumerate(samples):    \n",
        "        overlap, weight_sum = 0.0, 0.0\n",
        "        for index_sample, (sample, logp) in enumerate(zip(samples, log_probs)):           \n",
        "            if index_candidate == index_sample:\n",
        "                continue\n",
        "            sample_p = float(np.exp(logp))\n",
        "            weight_sum += sample_p\n",
        "            sample_overlap = rouge1_similarity(candidate, sample)\n",
        "            overlap += sample_p * sample_overlap\n",
        "        score = overlap / weight_sum\n",
        "        scores[index_candidate] = score\n",
        "    return scores"
      ],
      "metadata": {
        "id": "7KxiaRENt-C_"
      },
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def mbr_decode(sentence, n_samples, model=None, temperature=0.6):\n",
        "    samples, log_probs = generate_samples(sentence, n_samples, model, temperature)\n",
        "    scores = weighted_avg_overlap(samples, log_probs)\n",
        "    max_score_key = max(scores, key=scores.get)\n",
        "    return Counter({sample:score for sample, score in zip(samples, scores.values())})"
      ],
      "metadata": {
        "id": "ny5EYLFWt_5u"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "translated_sentence = mbr_decode(sentence, 2, transformer, 1)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "lzrxKRMAaOgr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "842658d3-4029-4398-a89b-50510cbdb9d7"
      },
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  I love you.\n",
            "Bengali:  Counter({'আমি তোমাকে ভালবাসি.. ..': 0.8571428571428571, 'তোমাকে আমি ভালবাসি।': 0.8571428571428571})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"How are you.\"\n",
        "translated_sentence = mbr_decode(sentence, 2, transformer, 1)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "id": "3ACvLvfouZz0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "689d5f3b-18a1-408f-8c04-53b7beb7588a"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  How are you.\n",
            "Bengali:  Counter({'কী মুন হয়ে বেড়াচ্ছে?': 0.6222222222222223, 'কী হচ্ছে জেনে নিন আপনি?': 0.6222222222222223})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"How are you.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hFnTZVktqKjJ",
        "outputId": "9e8d1892-d8b0-426f-f1cd-0a8602c1beda"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  How are you.\n",
            "Bengali:  Counter({'কীভাবে আছ?': 0.6003264957880456, 'কী ভাবে করে?': 0.5724018656412878, '“তুমি কেমন আছো? ”': 0.560174273069195, 'কী করে আছে বলুন।': 0.49856605353652655, 'কেমন হবে তা।': 0.48489607850601835, 'কী করে করে?': 0.46357777173301457, 'কী করিয়া?': 0.4348721424195965, 'কী ভাবে করে বলো তো।': 0.4148347483907275, 'কীভাবে?': 0.3884963689736238, 'তুমি কেমন আছ?': 0.35802342436828377})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I am tired.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QlRYyoz_qXcQ",
        "outputId": "a1cf1bbe-6abc-4b0d-9ba4-8016154429a4"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  I am tired.\n",
            "Bengali:  Counter({'আমি ক্লান্ত হয়ে গেছি।': 0.9364828505378495, 'ক্লান্ত হয়ে গেছি।': 0.8615868219234791, 'আমি ক্লান্ত হয়ে পড়ি।': 0.8395360379863634, 'ক্লান্ত হয়ে পড়ি।': 0.7482339505740715, 'ক্লান্ত হয়ে যাচ্ছি।': 0.7464140952552505, 'আমি ক্লান্ত।': 0.7026435083095889})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"Thank you everyone, it was a nice experience.\"\n",
        "translated_sentence = mbr_decode(sentence, 10, transformer)\n",
        "print(\"English: \", sentence)\n",
        "print(\"Bengali: \", translated_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EqW3eaYJqpyB",
        "outputId": "effa0748-5d7c-4b59-9167-aa7adb1e4c4c"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English:  Thank you everyone, it was a nice experience.\n",
            "Bengali:  Counter({'ধন্যবাদ জানিয়ে দিয়েছেন, সবার সঙ্গেই।': 0.8338564071212432, 'ধন্যবাদ জানিয়ে দিয়েছেন সবাইকে।': 0.784442273050588, 'ধন্যবাদ জানিয়ে দিয়েছেন সবার সঙ্গে।': 0.7783877530501729, 'ধন্যবাদ জানিয়ে দিয়েছেন সব রকমের অভিজ্ঞতা।': 0.7633445228776755, 'ধন্যবাদ জানিয়ে দিয়েছেন সবার অভিজ্ঞতার মধ্যে।': 0.729903828276666, 'ধন্যবাদ পেয়ে আপনি সবাই দারুণ অভিজ্ঞতা পেলেন।': 0.7198453501727136, 'ধন্যবাদ পেয়ে সবার জন্য দারুণ অভিজ্ঞতা।': 0.6959505036454724, 'ধন্যবাদ পেয়ে এই অভিজ্ঞতা দারুণ।': 0.6879594081790569, 'ধন্যবাদ পেয়ে সবাই খুশি।': 0.6788009157518129, 'কৃতজ্ঞতাবশত এ অভিজ্ঞতা দারুণ।': 0.45634586639586283})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model1 = Transformer(dropout = 0.3)\n",
        "model1.load_weights(\"model_weights-2l-8h-loss-changed\")"
      ],
      "metadata": {
        "id": "ia72Kt_Qu50s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ea350cc4-fc13-4104-fb36-933875833de0"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f292c67be50>"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"I love you.\"\n",
        "print(\"English Sentence: \", sentence)\n",
        "print(\"Bengali Sentence: \", greedy_translate(model1, sentence))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h6KLJHCXclF0",
        "outputId": "3b5c981e-a6d7-4f02-f6ea-5212457c4f00"
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English Sentence:  I love you.\n",
            "Bengali Sentence:  আমি তোমাকে ভালবাসি।\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "FcWrkrU8cuxz"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}