seq2seq_with_attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Seq2Seq_with_attention.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ihl1YwMaRRum",
        "colab_type": "text"
      },
      "source": [
        "# Sequence to Sequence attention model for machine translation\n",
        "\n",
        "This notebook trains a sequence to sequence (seq2seq) model with two different attentions implemented for Spanish to English translation.\n",
        "\n",
        "The codes are built on TensorFlow Core tutorials: https://www.tensorflow.org/tutorials/text/nmt_with_attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rfbpjs2fgzuT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9581a849-f5ea-42a5-855e-fa86b795759b"
      },
      "source": [
        "import tensorflow as tf\n",
        "print(tf.__version__)\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.ticker as ticker\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import unicodedata\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import io\n",
        "import time"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hOsMShvlg4ua",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "\n",
        "# Load data set\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SDa3tH1hRyNq",
        "colab_type": "text"
      },
      "source": [
        "*   Clean the sentences by removing special characters.\n",
        "*   Add a start and end token to each sentence.\n",
        "*   Create a word index and reverse word index (dictionaries mapping from word → id and id → word).\n",
        "*   Pad each sentence to a maximum length."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwlD7yEZhM72",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "d45ea8e2-1a20-4851-e544-581f9cc4db9e"
      },
      "source": [
        "# Download the file\n",
        "path_to_zip = tf.keras.utils.get_file(\n",
        "    'spa-eng.zip', origin='http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip',\n",
        "    extract=True)\n",
        "\n",
        "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\""
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip\n",
            "2646016/2638744 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DKjEWl6WhZyk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Converts the unicode file to ascii\n",
        "def unicode_to_ascii(s):\n",
        "  return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
        "      if unicodedata.category(c) != 'Mn')\n",
        "\n",
        "\n",
        "def preprocess_sentence(w):\n",
        "  w = unicode_to_ascii(w.lower().strip())\n",
        "\n",
        "  # creating a space between a word and the punctuation following it\n",
        "  # eg: \"he is a boy.\" => \"he is a boy .\"\n",
        "  # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
        "  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
        "  w = re.sub(r'[\" \"]+', \" \", w)\n",
        "\n",
        "  # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\",\"¿\")\n",
        "  w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
        "\n",
        "  # remove extra space\n",
        "  w = w.strip()\n",
        "\n",
        "  # adding a start and an end token to the sentence\n",
        "  # so that the model know when to start and stop predicting.\n",
        "  w = '<start> ' + w + ' <end>'\n",
        "  return w"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xc58-K0XhdCM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "fee8f365-72e6-41c0-8036-083c3f47fd12"
      },
      "source": [
        "en_sentence = u\"May I borrow this @ book?\"\n",
        "sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n",
        "print(preprocess_sentence(en_sentence))\n",
        "print(preprocess_sentence(sp_sentence))\n",
        "print(preprocess_sentence(sp_sentence).encode(\"UTF-8\"))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<start> may i borrow this book ? <end>\n",
            "<start> ¿ puedo tomar prestado este libro ? <end>\n",
            "b'<start> \\xc2\\xbf puedo tomar prestado este libro ? <end>'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Hw9ct4OhsRA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "d8b2bed5-2d5c-4d0f-b94d-c52a60ebbc5a"
      },
      "source": [
        "# Return word pairs in the format: [ENGLISH, SPANISH]\n",
        "def create_dataset(path, num_examples):\n",
        "  lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n",
        "  word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
        "  return zip(*word_pairs)\n",
        "\n",
        "en, sp = create_dataset(path_to_file, None)\n",
        "print(en[-1])\n",
        "print(sp[-1])\n",
        "print(len(en), len(sp))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<start> if you want to sound like a native speaker , you must be willing to practice saying the same sentence over and over in the same way that banjo players practice the same phrase over and over until they can play it correctly and at the desired tempo . <end>\n",
            "<start> si quieres sonar como un hablante nativo , debes estar dispuesto a practicar diciendo la misma frase una y otra vez de la misma manera en que un musico de banjo practica el mismo fraseo una y otra vez hasta que lo puedan tocar correctamente y en el tiempo esperado . <end>\n",
            "118964 118964\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZCAYRuTlh5DY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Tokenize the sentence into list of words(integers) and pad the sequence to the same length\n",
        "def tokenize(lang):\n",
        "  lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n",
        "      filters='')\n",
        "  lang_tokenizer.fit_on_texts(lang)\n",
        "\n",
        "  tensor = lang_tokenizer.texts_to_sequences(lang)\n",
        "\n",
        "  tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n",
        "                                                         padding='post')\n",
        "  return tensor, lang_tokenizer"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e13Aa8yliFkp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def load_dataset(path, num_examples=None):\n",
        "  # creating cleaned input, output pairs\n",
        "  targ_lang, inp_lang = create_dataset(path, num_examples)\n",
        "\n",
        "  input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n",
        "  target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n",
        "\n",
        "  return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EasB_FLig5c",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8f8f1403-163e-4125-c227-cce22bd8294b"
      },
      "source": [
        "# Try experimenting with the size of that dataset\n",
        "num_examples = 30000\n",
        "input_tensor, target_tensor, inp_lang, targ_lang = load_dataset(path_to_file, num_examples)\n",
        "\n",
        "# Calculate max_length of the target tensors\n",
        "max_length_targ, max_length_inp = target_tensor.shape[1], input_tensor.shape[1]\n",
        "print(max_length_targ, max_length_inp)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "11 16\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6oS6mqluirWG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "7fd7b07d-2593-4709-87aa-7e09f7a9472b"
      },
      "source": [
        "# Creating training and validation sets using an 80-20 split\n",
        "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
        "\n",
        "# Show length\n",
        "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))\n",
        "print(input_tensor_train[0])\n",
        "print(target_tensor_train[0])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "24000 24000 6000 6000\n",
            "[  1 134  10   4  44  41   3   2   0   0   0   0   0   0   0   0]\n",
            "[ 1  4 95  5 80 45  3  2  0  0  0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y4McPf16iuXH",
        "colab_type": "text"
      },
      "source": [
        "# Create a tf.data datasest"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lckoCvxAjj3U",
        "colab_type": "text"
      },
      "source": [
        "The tf.data.Dataset API supports writing descriptive and efficient input pipelines. Dataset usage follows a common pattern:\n",
        "\n",
        "\n",
        "*   Create a source dataset from your input data.\n",
        "*   Apply dataset transformations to preprocess the data.\n",
        "*   Iterate over the dataset and process the elements.\n",
        "\n",
        "Iteration happens in a streaming fashion, so the full dataset does not need to fit into memory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AK3NE1vUityv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c04a5d40-26fa-4ac3-ca41-f137568711bf"
      },
      "source": [
        "# Configuration \n",
        "BUFFER_SIZE = len(input_tensor_train)\n",
        "BATCH_SIZE = 64\n",
        "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
        "steps_per_epoch_val = len(input_tensor_val)//BATCH_SIZE\n",
        "embedding_dim = 256  # for word embedding\n",
        "units = 1024  # dimensionality of the output space of RNN\n",
        "vocab_inp_size = len(inp_lang.word_index)+1\n",
        "vocab_tar_size = len(targ_lang.word_index)+1\n",
        "\n",
        "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
        "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n",
        "validation_dataset = tf.data.Dataset.from_tensor_slices((input_tensor_val, target_tensor_val)).shuffle(BUFFER_SIZE)\n",
        "validation_dataset = validation_dataset.batch(BATCH_SIZE, drop_remainder=True)\n",
        "\n",
        "example_input_batch, example_target_batch = next(iter(dataset))\n",
        "example_input_batch.shape, example_target_batch.shape"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([64, 16]), TensorShape([64, 11]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jzo3ZvYpv4KE",
        "colab_type": "text"
      },
      "source": [
        "# Basic seq2seq model: encoder and decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ex8ORksZcTbV",
        "colab_type": "text"
      },
      "source": [
        "Model groups layers into an object with training and inference features. Two ways to define tf model:\n",
        "\n",
        "![alt text](https://i.ibb.co/c8JX8Cc/tf-Model.jpg)\n",
        "\n",
        "Basic sequence to sequence model without attention:\n",
        "![alt text](https://i.ibb.co/QN0tyMp/seq2seq.jpg)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFjq9Ta3wA8F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Encoder(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
        "    super(Encoder, self).__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.enc_units = enc_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.enc_units,\n",
        "                                   return_sequences=True,  # Whether to return the last output in the output sequence, or the full sequence. \n",
        "                                   return_state=True,  # Whether to return the last state in addition to the output.\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "\n",
        "  def call(self, x, hidden):\n",
        "    x = self.embedding(x)\n",
        "    output, state = self.gru(x, initial_state = hidden)\n",
        "    return output, state\n",
        "\n",
        "  def initialize_hidden_state(self):\n",
        "    return tf.zeros((self.batch_sz, self.enc_units))"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T4GoRSHSwScu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "4117b3c8-8007-4769-a153-675f1a098046"
      },
      "source": [
        "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
        "\n",
        "# sample input\n",
        "sample_hidden = encoder.initialize_hidden_state()\n",
        "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
        "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
        "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Encoder output shape: (batch size, sequence length, units) (64, 16, 1024)\n",
            "Encoder Hidden state shape: (batch size, units) (64, 1024)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sy6ualLHwYzY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Decoder(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
        "    super(Decoder, self).__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.dec_units = dec_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "\n",
        "\n",
        "  def call(self, x, hidden):\n",
        "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
        "    x = self.embedding(x)\n",
        "\n",
        "    # passing the concatenated vector to the GRU\n",
        "    output, state = self.gru(x, initial_state = hidden)\n",
        "\n",
        "    # output shape == (batch_size * 1, hidden_size)\n",
        "    output = tf.reshape(output, (-1, output.shape[2]))\n",
        "\n",
        "    # output shape == (batch_size, vocab)\n",
        "    x = self.fc(output)\n",
        "    return x, state"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rUXzFdWhhu1D",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "af254a07-8a65-46d6-eade-82329c482941"
      },
      "source": [
        "tf.reshape([[1,2,3],[4,5,6]], (-1, 2))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3, 2), dtype=int32, numpy=\n",
              "array([[1, 2],\n",
              "       [3, 4],\n",
              "       [5, 6]], dtype=int32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k_z9vs5U06hk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8378e62d-e45e-4b2c-fc61-3109d2cea71b"
      },
      "source": [
        "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
        "\n",
        "sample_decoder_output, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n",
        "                                      sample_hidden)\n",
        "\n",
        "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Decoder output shape: (batch_size, vocab size) (64, 4935)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZBGeYTE14Oij",
        "colab_type": "text"
      },
      "source": [
        "# Dot-product attention\n",
        "\n",
        "![alt text](https://i.ibb.co/TvhM1Z2/attention.jpg)\n",
        "\n",
        "![alt text](https://i.ibb.co/bvrcptV/dotproduct.jpg)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4vqqDed3xYH-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DotProductAttention(tf.keras.layers.Layer):\n",
        "  def call(self, query, values):\n",
        "    # query hidden state shape == (batch_size, hidden size)\n",
        "    # query_with_time_axis shape == (batch_size, 1, hidden size)\n",
        "    # values shape == (batch_size, max_len, hidden size)\n",
        "    # we are doing this to broadcast addition along the time axis to calculate the score\n",
        "    query_with_time_axis = tf.expand_dims(query, 1)\n",
        "\n",
        "    # inner product, score shape == (batch_size, max_length, 1)\n",
        "    score = query_with_time_axis * values\n",
        "    score = tf.reduce_sum(score, axis=2)\n",
        "    score = tf.expand_dims(score, 2)\n",
        "    \n",
        "    # attention_weights shape == (batch_size, max_length, 1)\n",
        "    attention_weights = tf.nn.softmax(score, axis=1)\n",
        "\n",
        "    # context_vector shape after sum == (batch_size, hidden_size)\n",
        "    context_vector = attention_weights * values\n",
        "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "\n",
        "    return context_vector, attention_weights"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JlQqI7zA2MTO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "628955f9-4814-4784-aa8c-9894045abd0a"
      },
      "source": [
        "attention_layer = DotProductAttention()\n",
        "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
        "\n",
        "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
        "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention result shape: (batch size, units) (64, 1024)\n",
            "Attention weights shape: (batch_size, sequence_length, 1) (64, 16, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YSWViCN635g8",
        "colab_type": "text"
      },
      "source": [
        "# Additive attention\n",
        "\n",
        "![alt text](https://i.ibb.co/BqDYNP1/additive.jpg)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U1-eiiJ_xyoM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BahdanauAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, units):\n",
        "    super(BahdanauAttention, self).__init__()\n",
        "    self.W1 = tf.keras.layers.Dense(units)\n",
        "    self.W2 = tf.keras.layers.Dense(units)\n",
        "    self.V = tf.keras.layers.Dense(1)\n",
        "\n",
        "  def call(self, query, values):\n",
        "    # query hidden state shape == (batch_size, hidden size)\n",
        "    # query_with_time_axis shape == (batch_size, 1, hidden size)\n",
        "    # values shape == (batch_size, max_len, hidden size)\n",
        "    query_with_time_axis = tf.expand_dims(query, 1)\n",
        "\n",
        "    # score shape == (batch_size, max_length, 1)\n",
        "    # we get 1 at the last axis because we are applying score to self.V\n",
        "    # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
        "    score = self.V(tf.nn.tanh(\n",
        "        self.W1(values) + self.W2(query_with_time_axis)))\n",
        "\n",
        "    # attention_weights shape == (batch_size, max_length, 1)\n",
        "    attention_weights = tf.nn.softmax(score, axis=1)\n",
        "\n",
        "    # context_vector shape after sum == (batch_size, hidden_size)\n",
        "    context_vector = attention_weights * values\n",
        "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "\n",
        "    return context_vector, attention_weights"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7cnv4ILg87cM",
        "colab_type": "text"
      },
      "source": [
        "# Decoder layer with attention\n",
        "\n",
        "![alt text](https://i.ibb.co/ZM25Zvv/Context-Vector.jpg)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bXqWOEGh9BAm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DecoderWithAttention(tf.keras.Model):\n",
        "  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz, attention_layer = None):\n",
        "    super(DecoderWithAttention, self).__init__()\n",
        "    self.batch_sz = batch_sz\n",
        "    self.dec_units = dec_units\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
        "\n",
        "    # used for attention\n",
        "    self.attention = attention_layer\n",
        "\n",
        "  def call(self, x, hidden, enc_output):\n",
        "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
        "    x = self.embedding(x)\n",
        "    attention_weights = None\n",
        "    \n",
        "    if self.attention:\n",
        "      # enc_output shape == (batch_size, max_length, hidden_size)\n",
        "      context_vector, attention_weights = self.attention(hidden, enc_output)\n",
        "      # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
        "      x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "\n",
        "    # passing the concatenated vector to the GRU\n",
        "    output, state = self.gru(x, initial_state = hidden)\n",
        "\n",
        "    # output shape == (batch_size * 1, hidden_size)\n",
        "    output = tf.reshape(output, (-1, output.shape[2]))\n",
        "\n",
        "    # output shape == (batch_size, vocab)\n",
        "    x = self.fc(output)\n",
        "\n",
        "    return x, state, attention_weights"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fK2x4JEx1Gji",
        "colab_type": "text"
      },
      "source": [
        "# Define loss function\n",
        "\n",
        "Cross-entropy loss, or log loss, measures the performance of a classification model whose output is a probability value between 0 and 1. Cross-entropy loss increases as the predicted probability diverges from the actual label. \n",
        "![alt text](https://i.ibb.co/GtD1vc9/cross-entropy.png)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F9oozBiV1Khj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "\n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0)) \n",
        "  loss_ = loss_object(real, pred)\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "  return tf.reduce_mean(loss_)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFLLiTBgnwG-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "12724107-7981-43cf-fc9a-cd9d60fa872b"
      },
      "source": [
        "print(loss_object([1,2],[[0,0.6,0.3,0.1],[0,0.6,0.3,0.1]]))\n",
        "print(loss_function([1,2],[[0,0.6,0.3,0.1],[0,0.6,0.3,0.1]]))"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor([1.063386  1.3633859], shape=(2,), dtype=float32)\n",
            "tf.Tensor(1.2133859, shape=(), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h2NpO5aG1UDu",
        "colab_type": "text"
      },
      "source": [
        "# Training\n",
        "\n",
        "@tf.function\n",
        "In TensorFlow 2, eager execution is turned on by default. The user interface is intuitive and flexible (running one-off operations is much easier and faster), but this can come at the expense of performance and deployability. It is recommended to debug in eager mode, then decorate with @tf.function for better performance.\n",
        "\n",
        "In TensorFlow 2.0, users should refactor their code into smaller functions which are called as needed. In general, it's not necessary to decorate each of these smaller functions with tf.function; only use tf.function to decorate high-level computations - for example, one step of training, or the forward pass of your model.\n",
        "\n",
        "TensorFlow provides the tf.GradientTape API for automatic differentiation; that is, computing the gradient of a computation with respect to some inputs, usually tf.Variables. TensorFlow \"records\" relevant operations executed inside the context of a tf.GradientTape onto a \"tape\". TensorFlow then uses that tape to compute the gradients of a \"recorded\" computation using reverse mode differentiation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mz6c3l931TC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "\n",
        "def get_train_step_func():\n",
        "\n",
        "  @tf.function\n",
        "  def train_step(inp, targ, enc_hidden, encoder, decoder):\n",
        "    loss = 0\n",
        "\n",
        "    with tf.GradientTape() as tape: # for automatic differentiation\n",
        "      enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
        "\n",
        "      dec_hidden = enc_hidden\n",
        "\n",
        "      dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "\n",
        "      # Teacher forcing - feeding the target as the next input\n",
        "      for t in range(1, targ.shape[1]):\n",
        "        # passing enc_output to the decoder\n",
        "        predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "\n",
        "        loss += loss_function(targ[:, t], predictions)\n",
        "\n",
        "        # using teacher forcing\n",
        "        dec_input = tf.expand_dims(targ[:, t], 1)\n",
        "\n",
        "    batch_loss = (loss / int(targ.shape[1]))\n",
        "\n",
        "    variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "\n",
        "    gradients = tape.gradient(loss, variables)\n",
        "\n",
        "    optimizer.apply_gradients(zip(gradients, variables))\n",
        "\n",
        "    return batch_loss\n",
        "    \n",
        "  return train_step\n",
        "    "
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6zym-buZ_TEg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def caculate_validation_loss(inp, targ, enc_hidden, encoder, decoder):\n",
        "  loss = 0\n",
        "  enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
        "  dec_hidden = enc_hidden\n",
        "  dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "\n",
        "  # Teacher forcing - feeding the target as the next input\n",
        "  for t in range(1, targ.shape[1]):\n",
        "    predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
        "    loss += loss_function(targ[:, t], predictions)\n",
        "    dec_input = tf.expand_dims(targ[:, t], 1)\n",
        "\n",
        "  loss = loss / int(targ.shape[1])\n",
        "  return loss"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kMo0_PVaJiP7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def training_seq2seq(epochs, attention):\n",
        "  encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
        "  decoder = DecoderWithAttention(vocab_tar_size, embedding_dim, units, BATCH_SIZE, attention)\n",
        "  train_step_func = get_train_step_func()\n",
        "  training_loss = []\n",
        "  validation_loss = []\n",
        "\n",
        "  for epoch in range(epochs):\n",
        "    start = time.time()\n",
        "    enc_hidden = encoder.initialize_hidden_state()\n",
        "    total_loss = 0\n",
        "\n",
        "    for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
        "      batch_loss = train_step_func(inp, targ, enc_hidden, encoder, decoder)\n",
        "      total_loss += batch_loss\n",
        "\n",
        "      if batch % 100 == 0:\n",
        "        print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, batch_loss))\n",
        "        \n",
        "    enc_hidden = encoder.initialize_hidden_state()\n",
        "    total_val_loss = 0\n",
        "    for (batch, (inp, targ)) in enumerate(validation_dataset.take(steps_per_epoch)):\n",
        "      val_loss = caculate_validation_loss(inp, targ, enc_hidden, encoder, decoder)\n",
        "      total_val_loss += val_loss\n",
        "\n",
        "    training_loss.append(total_loss / steps_per_epoch)\n",
        "    validation_loss.append(total_val_loss / steps_per_epoch_val)\n",
        "    print('Epoch {} Loss {:.4f} Validation Loss {:.4f}'.format(epoch + 1,\n",
        "                                        training_loss[-1], validation_loss[-1]))\n",
        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n",
        "  return encoder, decoder, training_loss, validation_loss"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r96cK-kAVLbG",
        "colab_type": "text"
      },
      "source": [
        "## Training seq2seq without attention\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4knY8jaQ1gcO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "04dca603-e126-493e-e974-7150e48aeeef"
      },
      "source": [
        "epochs = 10\n",
        "attention = None\n",
        "\n",
        "print(\"Running seq2seq model without attention\")\n",
        "encoder, decoder, training_loss, validation_loss = training_seq2seq(epochs, attention)\n",
        "\n",
        "tloss = training_loss\n",
        "vloss = validation_loss"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running seq2seq model without attention\n",
            "Epoch 1 Batch 0 Loss 4.6869\n",
            "Epoch 1 Batch 100 Loss 2.1395\n",
            "Epoch 1 Batch 200 Loss 1.8686\n",
            "Epoch 1 Batch 300 Loss 1.6464\n",
            "Epoch 1 Loss 1.9497 Validation Loss 1.5812\n",
            "Time taken for 1 epoch 37.284785747528076 sec\n",
            "\n",
            "Epoch 2 Batch 0 Loss 1.5578\n",
            "Epoch 2 Batch 100 Loss 1.4306\n",
            "Epoch 2 Batch 200 Loss 1.4045\n",
            "Epoch 2 Batch 300 Loss 1.3330\n",
            "Epoch 2 Loss 1.3965 Validation Loss 1.3412\n",
            "Time taken for 1 epoch 29.765738487243652 sec\n",
            "\n",
            "Epoch 3 Batch 0 Loss 1.1735\n",
            "Epoch 3 Batch 100 Loss 1.2294\n",
            "Epoch 3 Batch 200 Loss 1.1231\n",
            "Epoch 3 Batch 300 Loss 1.0149\n",
            "Epoch 3 Loss 1.1215 Validation Loss 1.2057\n",
            "Time taken for 1 epoch 29.904013872146606 sec\n",
            "\n",
            "Epoch 4 Batch 0 Loss 0.8761\n",
            "Epoch 4 Batch 100 Loss 0.8823\n",
            "Epoch 4 Batch 200 Loss 0.8799\n",
            "Epoch 4 Batch 300 Loss 0.9043\n",
            "Epoch 4 Loss 0.9073 Validation Loss 1.0975\n",
            "Time taken for 1 epoch 30.0055148601532 sec\n",
            "\n",
            "Epoch 5 Batch 0 Loss 0.6926\n",
            "Epoch 5 Batch 100 Loss 0.7090\n",
            "Epoch 5 Batch 200 Loss 0.7810\n",
            "Epoch 5 Batch 300 Loss 0.7730\n",
            "Epoch 5 Loss 0.7218 Validation Loss 1.0257\n",
            "Time taken for 1 epoch 29.86213254928589 sec\n",
            "\n",
            "Epoch 6 Batch 0 Loss 0.4796\n",
            "Epoch 6 Batch 100 Loss 0.5501\n",
            "Epoch 6 Batch 200 Loss 0.6577\n",
            "Epoch 6 Batch 300 Loss 0.5363\n",
            "Epoch 6 Loss 0.5660 Validation Loss 0.9892\n",
            "Time taken for 1 epoch 29.855602502822876 sec\n",
            "\n",
            "Epoch 7 Batch 0 Loss 0.4077\n",
            "Epoch 7 Batch 100 Loss 0.3633\n",
            "Epoch 7 Batch 200 Loss 0.4509\n",
            "Epoch 7 Batch 300 Loss 0.4631\n",
            "Epoch 7 Loss 0.4390 Validation Loss 0.9606\n",
            "Time taken for 1 epoch 30.220492839813232 sec\n",
            "\n",
            "Epoch 8 Batch 0 Loss 0.2913\n",
            "Epoch 8 Batch 100 Loss 0.3730\n",
            "Epoch 8 Batch 200 Loss 0.3265\n",
            "Epoch 8 Batch 300 Loss 0.3581\n",
            "Epoch 8 Loss 0.3365 Validation Loss 0.9582\n",
            "Time taken for 1 epoch 30.660627841949463 sec\n",
            "\n",
            "Epoch 9 Batch 0 Loss 0.2343\n",
            "Epoch 9 Batch 100 Loss 0.2408\n",
            "Epoch 9 Batch 200 Loss 0.2485\n",
            "Epoch 9 Batch 300 Loss 0.2376\n",
            "Epoch 9 Loss 0.2550 Validation Loss 0.9644\n",
            "Time taken for 1 epoch 30.542608737945557 sec\n",
            "\n",
            "Epoch 10 Batch 0 Loss 0.2025\n",
            "Epoch 10 Batch 100 Loss 0.1809\n",
            "Epoch 10 Batch 200 Loss 0.1757\n",
            "Epoch 10 Batch 300 Loss 0.2049\n",
            "Epoch 10 Loss 0.1909 Validation Loss 0.9752\n",
            "Time taken for 1 epoch 30.083377361297607 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C2w0wAvhVUtq",
        "colab_type": "text"
      },
      "source": [
        "## Training seq2seq with dot product attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "alodSzmpX77O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "c5c651e8-9c7a-4bfa-86d3-ef331b47f62d"
      },
      "source": [
        "attention = DotProductAttention()\n",
        "print(\"Running seq2seq model with dot product attention\")\n",
        "encoder_dp, decoder_dp, training_loss, validation_loss = training_seq2seq(epochs, attention)\n",
        "\n",
        "tloss = np.vstack((tloss, training_loss))\n",
        "vloss = np.vstack((vloss, validation_loss))"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running seq2seq model with dot product attention\n",
            "Epoch 1 Batch 0 Loss 4.6264\n",
            "Epoch 1 Batch 100 Loss 2.4235\n",
            "Epoch 1 Batch 200 Loss 2.1550\n",
            "Epoch 1 Batch 300 Loss 1.8926\n",
            "Epoch 1 Loss 2.4485 Validation Loss 1.9181\n",
            "Time taken for 1 epoch 42.6515851020813 sec\n",
            "\n",
            "Epoch 2 Batch 0 Loss 1.8629\n",
            "Epoch 2 Batch 100 Loss 1.7040\n",
            "Epoch 2 Batch 200 Loss 1.4752\n",
            "Epoch 2 Batch 300 Loss 1.5064\n",
            "Epoch 2 Loss 1.6576 Validation Loss 1.5580\n",
            "Time taken for 1 epoch 34.465418100357056 sec\n",
            "\n",
            "Epoch 3 Batch 0 Loss 1.3883\n",
            "Epoch 3 Batch 100 Loss 1.3110\n",
            "Epoch 3 Batch 200 Loss 1.4096\n",
            "Epoch 3 Batch 300 Loss 1.3529\n",
            "Epoch 3 Loss 1.3195 Validation Loss 1.3515\n",
            "Time taken for 1 epoch 34.06086468696594 sec\n",
            "\n",
            "Epoch 4 Batch 0 Loss 1.1593\n",
            "Epoch 4 Batch 100 Loss 1.0606\n",
            "Epoch 4 Batch 200 Loss 0.9367\n",
            "Epoch 4 Batch 300 Loss 0.9321\n",
            "Epoch 4 Loss 1.0839 Validation Loss 1.2153\n",
            "Time taken for 1 epoch 34.23249864578247 sec\n",
            "\n",
            "Epoch 5 Batch 0 Loss 0.8570\n",
            "Epoch 5 Batch 100 Loss 0.9223\n",
            "Epoch 5 Batch 200 Loss 0.9343\n",
            "Epoch 5 Batch 300 Loss 0.7972\n",
            "Epoch 5 Loss 0.8963 Validation Loss 1.1206\n",
            "Time taken for 1 epoch 34.13781118392944 sec\n",
            "\n",
            "Epoch 6 Batch 0 Loss 0.6813\n",
            "Epoch 6 Batch 100 Loss 0.7029\n",
            "Epoch 6 Batch 200 Loss 0.7612\n",
            "Epoch 6 Batch 300 Loss 0.7500\n",
            "Epoch 6 Loss 0.7414 Validation Loss 1.0458\n",
            "Time taken for 1 epoch 33.956538677215576 sec\n",
            "\n",
            "Epoch 7 Batch 0 Loss 0.6269\n",
            "Epoch 7 Batch 100 Loss 0.7352\n",
            "Epoch 7 Batch 200 Loss 0.6241\n",
            "Epoch 7 Batch 300 Loss 0.5669\n",
            "Epoch 7 Loss 0.6110 Validation Loss 0.9914\n",
            "Time taken for 1 epoch 33.77721953392029 sec\n",
            "\n",
            "Epoch 8 Batch 0 Loss 0.4866\n",
            "Epoch 8 Batch 100 Loss 0.4470\n",
            "Epoch 8 Batch 200 Loss 0.4653\n",
            "Epoch 8 Batch 300 Loss 0.5365\n",
            "Epoch 8 Loss 0.5020 Validation Loss 0.9507\n",
            "Time taken for 1 epoch 33.88840341567993 sec\n",
            "\n",
            "Epoch 9 Batch 0 Loss 0.4635\n",
            "Epoch 9 Batch 100 Loss 0.4269\n",
            "Epoch 9 Batch 200 Loss 0.4993\n",
            "Epoch 9 Batch 300 Loss 0.4623\n",
            "Epoch 9 Loss 0.4111 Validation Loss 0.9175\n",
            "Time taken for 1 epoch 33.621456146240234 sec\n",
            "\n",
            "Epoch 10 Batch 0 Loss 0.4042\n",
            "Epoch 10 Batch 100 Loss 0.3281\n",
            "Epoch 10 Batch 200 Loss 0.3313\n",
            "Epoch 10 Batch 300 Loss 0.2708\n",
            "Epoch 10 Loss 0.3350 Validation Loss 0.8871\n",
            "Time taken for 1 epoch 33.877196073532104 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sQKjqurFVY0u",
        "colab_type": "text"
      },
      "source": [
        "## Training seq2seq with Bahdanau attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7TUKivtyYix6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5dd50265-6e67-40da-86a3-a9e31c073f41"
      },
      "source": [
        "epochs = 10\n",
        "\n",
        "attention = BahdanauAttention(units)\n",
        "print(\"Running seq2seq model with Bahdanau attention\")\n",
        "encoder_bah, decoder_bah, training_loss, validation_loss = training_seq2seq(epochs, attention)\n",
        "\n",
        "tloss = np.vstack((tloss, training_loss))\n",
        "vloss = np.vstack((vloss, validation_loss))"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running seq2seq model with Bahdanau attention\n",
            "Epoch 1 Batch 0 Loss 4.5053\n",
            "Epoch 1 Batch 100 Loss 2.4014\n",
            "Epoch 1 Batch 200 Loss 1.8638\n",
            "Epoch 1 Batch 300 Loss 1.8508\n",
            "Epoch 1 Loss 2.2304 Validation Loss 1.6459\n",
            "Time taken for 1 epoch 49.266451835632324 sec\n",
            "\n",
            "Epoch 2 Batch 0 Loss 1.5681\n",
            "Epoch 2 Batch 100 Loss 1.4242\n",
            "Epoch 2 Batch 200 Loss 1.2935\n",
            "Epoch 2 Batch 300 Loss 1.2440\n",
            "Epoch 2 Loss 1.3518 Validation Loss 1.2219\n",
            "Time taken for 1 epoch 41.31578230857849 sec\n",
            "\n",
            "Epoch 3 Batch 0 Loss 1.1384\n",
            "Epoch 3 Batch 100 Loss 0.9462\n",
            "Epoch 3 Batch 200 Loss 1.0665\n",
            "Epoch 3 Batch 300 Loss 0.8158\n",
            "Epoch 3 Loss 0.9383 Validation Loss 0.9957\n",
            "Time taken for 1 epoch 41.691455125808716 sec\n",
            "\n",
            "Epoch 4 Batch 0 Loss 0.6284\n",
            "Epoch 4 Batch 100 Loss 0.6292\n",
            "Epoch 4 Batch 200 Loss 0.6123\n",
            "Epoch 4 Batch 300 Loss 0.6204\n",
            "Epoch 4 Loss 0.6468 Validation Loss 0.8762\n",
            "Time taken for 1 epoch 42.22861170768738 sec\n",
            "\n",
            "Epoch 5 Batch 0 Loss 0.4090\n",
            "Epoch 5 Batch 100 Loss 0.4194\n",
            "Epoch 5 Batch 200 Loss 0.4316\n",
            "Epoch 5 Batch 300 Loss 0.5111\n",
            "Epoch 5 Loss 0.4441 Validation Loss 0.8132\n",
            "Time taken for 1 epoch 42.191874742507935 sec\n",
            "\n",
            "Epoch 6 Batch 0 Loss 0.3097\n",
            "Epoch 6 Batch 100 Loss 0.2659\n",
            "Epoch 6 Batch 200 Loss 0.2879\n",
            "Epoch 6 Batch 300 Loss 0.2659\n",
            "Epoch 6 Loss 0.3054 Validation Loss 0.7782\n",
            "Time taken for 1 epoch 41.95083737373352 sec\n",
            "\n",
            "Epoch 7 Batch 0 Loss 0.2015\n",
            "Epoch 7 Batch 100 Loss 0.1861\n",
            "Epoch 7 Batch 200 Loss 0.2536\n",
            "Epoch 7 Batch 300 Loss 0.2544\n",
            "Epoch 7 Loss 0.2145 Validation Loss 0.7608\n",
            "Time taken for 1 epoch 41.63440918922424 sec\n",
            "\n",
            "Epoch 8 Batch 0 Loss 0.1759\n",
            "Epoch 8 Batch 100 Loss 0.1875\n",
            "Epoch 8 Batch 200 Loss 0.1559\n",
            "Epoch 8 Batch 300 Loss 0.2081\n",
            "Epoch 8 Loss 0.1536 Validation Loss 0.7595\n",
            "Time taken for 1 epoch 41.56172251701355 sec\n",
            "\n",
            "Epoch 9 Batch 0 Loss 0.1397\n",
            "Epoch 9 Batch 100 Loss 0.0913\n",
            "Epoch 9 Batch 200 Loss 0.1051\n",
            "Epoch 9 Batch 300 Loss 0.1352\n",
            "Epoch 9 Loss 0.1152 Validation Loss 0.7651\n",
            "Time taken for 1 epoch 41.66077256202698 sec\n",
            "\n",
            "Epoch 10 Batch 0 Loss 0.0813\n",
            "Epoch 10 Batch 100 Loss 0.0750\n",
            "Epoch 10 Batch 200 Loss 0.1285\n",
            "Epoch 10 Batch 300 Loss 0.1169\n",
            "Epoch 10 Loss 0.0897 Validation Loss 0.7695\n",
            "Time taken for 1 epoch 42.043405294418335 sec\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e5Yk-AZ4h3Hb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "outputId": "30828149-f99d-47d9-eee5-67731d16df95"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "ax = plt.subplot(111) \n",
        "t = np.arange(1, epochs+1)\n",
        "\n",
        "for i in range(0, vloss.shape[0]):\n",
        "  line, = plt.plot(t, vloss[i,:], lw=2)\n",
        "\n",
        "ax.legend(('No attention', 'Dot product', 'Bahdanau'))\n",
        "ax.set_title(\"Validation loss\")\n"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 1.0, 'Validation loss')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUVf7H8feZyaQ3SAgktNBbSKF3ERCxIqKySlVXVgHb2nVX+emuuruWVVFZC1JExRVFV1ERAekKSA+hh5IESAKk9zm/P+5kCBCSkGRyU76v5+HJnXvv3Ptl1v3kcO6Zc5TWGiGEEHWfxewChBBCVA8JdCGEqCck0IUQop6QQBdCiHpCAl0IIeoJCXQhhKgnJNBFnaCU0kqp9o7t2Uqpv1bk3ErcZ7xSalll6yzjukOVUser+7pClCSBLmqEUuoHpdTzpewfrZQ6oZRyq+i1tNb3aq1fqIaawh3h77y31nqh1npkVa8thBkk0EVNmQdMUEqpC/ZPBBZqrQtNqEmIekUCXdSUJUAQMLh4h1KqEXA9MF8p1UcptUEpdVYplaSUmqWUci/tQkqpuUqpv5V4/ZjjPYlKqbsuOPc6pdRWpVS6UuqYUmpmicOrHT/PKqUylVL9lVJTlFJrS7x/gFJqk1IqzfFzQIljq5RSLyil1imlMpRSy5RSwRX5MJRSXRzvP6uU2q2UurHEsWuVUrGOayYopR517A9WSn3reM9ppdQapZT8f1g4yX8MokZorXOAz4FJJXbfBsRprbcDRcDDQDDQHxgOTCvvukqpUcCjwFVAB2DEBadkOe4ZCFwH3KeUuslxbIjjZ6DW2ldrveGCazcGvgPexPhl9BrwnVIqqMRpdwB3AiGAu6OW8mq2Af8Dljnedz+wUCnVyXHKh8CftNZ+QASwwrH/EeA40ARoCjwNyNwdwkkCXdSkecAtSilPx+tJjn1orbdorTdqrQu11vHAf4ArKnDN24CPtNa7tNZZwMySB7XWq7TWO7XWdq31DuDTCl4XjF8A+7XWCxx1fQrEATeUOOcjrfW+Er+woitw3X6AL/Cy1jpfa70C+Ba43XG8AOiqlPLXWp/RWv9eYn8o0FprXaC1XqNlMiZRggS6qDFa67VACnCTUqod0Af4BEAp1dHRnXBCKZUOvIjRWi9PGHCsxOsjJQ8qpfoqpVYqpZKVUmnAvRW8bvG1j1yw7wjQvMTrEyW2szGCukI1a63tl7juWOBa4IhS6helVH/H/n8BB4BlSqlDSqknK/bXEA2FBLqoafMxWuYTgB+11icd+9/FaP120Fr7Y3QnXPgAtTRJQMsSr1tdcPwT4BugpdY6AJhd4rrltW4TgdYX7GsFJFSgrvKu2/KC/m/ndbXWm7TWozG6Y5ZgtPzRWmdorR/RWrcFbgT+rJQaXsVaRD0igS5q2nyMfu57cHS3OPgB6UCmUqozcF8Fr/c5MEUp1VUp5Q08d8FxP+C01jpXKdUHo8+7WDJgB9pe4tpLgY5KqTuUUm5KqXFAV4zukar4FaM1/7hSyqaUGorRjfOZUsrdMRY+QGtdgPGZ2AGUUtcrpdo7RgqlYTx3sJd+C9EQSaCLGuXoH18P+GC0nIs9ihG2GcD7wKIKXu974N8YDw4PcO4BYrFpwPNKqQzgWRytXcd7s4G/A+scI0f6XXDtVIxROI8AqcDjwPVa65SK1FZGzfkYAX4NRhfUO8AkrXWc45SJQLyj6+leYLxjfwdgOZAJbADe0VqvrEoton5R8kxFCCHqB2mhCyFEPSGBLoQQ9YQEuhBC1BMS6EIIUU9UeIa76hYcHKzDw8PNur0QQtRJW7ZsSdFaNyntmGmBHh4ezubNm826vRBC1ElKqQu/vewkXS5CCFFPSKALIUQ9IYEuhBD1hGl96EKI2qugoIDjx4+Tm5trdikNlqenJy1atMBms1X4PRLoQoiLHD9+HD8/P8LDw7l41UDhalprUlNTOX78OG3atKnw+6TLRQhxkdzcXIKCgiTMTaKUIigo6LL/hVQ3A71I1hMWwtUkzM1Vmc+/7gX61o/hjUg4FVf+uUII0YDUvUBP2ALpCbD8wnUMhBD1iVKKRx55xPn6lVdeYebMmVW+7rZt21i6dKnz9apVq1i/fn2lr3f27Fneeecd5+vExERuueWWKtVYWXUv0Ic+Be6+sO8HOLzG7GqEEC7i4eHBl19+SUpKldYTuYirAz0sLIwvvviiSjVWVt0LdN8QGPigsb3sL2CXFbiEqI/c3NyYOnUqr7/++kXH4uPjGTZsGJGRkQwfPpyjR49edM5vv/1G//79iYmJYcCAAezdu5f8/HyeffZZFi1aRHR0NP/4xz+YPXs2r7/+OtHR0axZs4bk5GTGjh1L79696d27N+vWrQNg5syZ3HXXXQwdOpS2bdvy5ptvAvDkk09y8OBBoqOjeeyxx4iPjyciIgIwHi7feeeddO/enZiYGFauNBaYmjt3LjfffDOjRo2iQ4cOPP7449XzmVXLVWpa/+mw6UNI2ga7FkPkrWZXJES9Ff7kdy65bvzL15V7zvTp04mMjLwo8O6//34mT57M5MmTmTNnDg888ABLliw575zOnTuzZs0a3NzcWL58OU8//TSLFy/m+eefZ/PmzcyaNQuAnJwcfH19efTRRwG44447ePjhhxk0aBBHjx7l6quvZs+ePQDExcWxcuVKMjIy6NSpE/fddx8vv/wyu3btYtu2bcbfKz7eWcPbb7+NUoqdO3cSFxfHyJEj2bdvH2D8S2Hr1q14eHjQqVMn7r//flq2LLne+eWrm4Hu7gPDnoFv7oefn4cuN4DN0+yqhBDVzN/fn0mTJvHmm2/i5eXl3L9hwwa+/PJLACZOnFhqCzctLY3Jkyezf/9+lFIUFBRU6J7Lly8nNjbW+To9PZ3MzEwArrvuOjw8PPDw8CAkJISTJ0+Wea21a9dy//33A8YvmNatWzsDffjw4QQEBADQtWtXjhw50kADHSB6PGx8F07Fwm/vwcAHzK5IiHqpIi1pV3rooYfo0aMHd95552W9769//StXXnklX331FfHx8QwdOrRC77Pb7WzcuBFPz4sbiR4eHs5tq9VKYWHlh1BX57WK1b0+9GIWK1z1vLG95hXIPm1uPUIIl2jcuDG33XYbH374oXPfgAED+OyzzwBYuHAhgwcPvuh9aWlpNG/eHDD6rIv5+fmRkZFxydcjR47krbfecr4u7kq5lAvfX9LgwYNZuHAhAPv27ePo0aN06tSpzOtVRd0NdID2I6DNFZCbBqtfMbsaIYSLPPLII+eNdnnrrbf46KOPiIyMZMGCBbzxxhsXvefxxx/nqaeeIiYm5rzW75VXXklsbCzR0dEsWrSIG264ga+++sr5UPTNN99k8+bNREZG0rVrV2bPnl1mbUFBQQwcOJCIiAgee+yx845NmzYNu91O9+7dGTduHHPnzj2vZV7dlNbaZRcvS69evXS1LHCRtB3+cwVY3GDGJmhc8XkPhBCl27NnD126dDG7jAavtP8dlFJbtNa9Sju/brfQAUKjIHIc2AuMB6RCCNFA1f1ABxj2F7B6wO4v4fgWs6sRQghT1I9AD2wJ/e41tn/6K5jUjSSEEGaqH4EOMOjP4NUYjqyDvd+bXY0QQtS4+hPoXoFwhePLBcufkyl2hRANTv0JdIBed0OjNpCyD36fZ3Y1QghRo+pXoLu5wwjHtLqrXoK80gf7CyFqP6vVSnR0NN26dSMqKopXX30VezmT8V04k6IrTZkypdKzKr744ovVXI2hfgU6QNeboEVvyEqGdW+aXY0QopK8vLzYtm0bu3fv5qeffuL777/n//7v/8p8T1UDXWtd7i+N6mBaoCul5iilTimldl3ieIBS6n9Kqe1Kqd1KqcubcKG6KQUj/2Zsb5gF6UmmliOEqLqQkBDee+89Zs2ahda61GlpL5wad9GiReddY+7cuYwePZqhQ4fSoUMH5y+H+Ph4OnXqxKRJk4iIiODYsWM89thjRERE0L17d+d1tNbMmDGDTp06MWLECE6dOuW8dnh4uPObrJs3b3bOG5OZmemsMzIyksWLF/Pkk0+Sk5NDdHQ048ePr9bPqSKTc80FZgHzL3F8OhCrtb5BKdUE2KuUWqi1zq+mGi9fq37Q+XqI+xZW/h1GzzKtFCHqvJkBLrpu2mWd3rZtW4qKijh16hQff/xxqdPSXjg17oV+++03du3ahbe3N7179+a6664jODiY/fv3M2/ePPr168fixYvZtm0b27dvJyUlhd69ezNkyBA2bNjA3r17iY2N5eTJk3Tt2pW77rqrzJpfeOEFAgIC2LlzJwBnzpxh7NixzJo1q9w5Yiqj3Ba61no1UNbMVxrwU8aKpr6Oc80fYjLi/4zpALYthJOx5Z8vhKgz1q5dy4QJE4CLp6Uty1VXXUVQUBBeXl7cfPPNrF27FoDWrVvTr18/57Vvv/12rFYrTZs25YorrmDTpk2sXr3auT8sLIxhw4aVe7/ly5czffp05+tGjRpV5q9bYdUxfe4s4BsgEfADxmmtS+2EUkpNBaYCtGrVqhpuXYbg9tDzTtj0Pvz0LEwwZ0koIeq8y2xJu8qhQ4ewWq2EhIRU+hpGu/Pi1z4+PlWqzc3Nzdn3npubW6VrVUV1PBS9GtgGhAHRwCyllH9pJ2qt39Na99Ja92rSpEk13LocVzwB7n5w4Cc4tMr19xNCuERycjL33nsvM2bMQCl1yWlpy5rKFuCnn37i9OnT5OTksGTJEgYOHHjROYMHD2bRokUUFRWRnJzM6tWr6dOnD0OGDHHuT0pKci4nB0Yf+pYtxrQjixcvdu6/6qqrePvtt52vz5w5A4DNZqvwghuXozoC/U7gS204ABwGOlfDdavOtwkMKl5/9K+y/qgQdUjxg8Nu3boxYsQIRo4cyXPPGcOSLzUt7YVT416oT58+jB07lsjISMaOHUuvXhdPWjhmzBgiIyOJiopi2LBh/POf/6RZs2aMGTOGDh060LVrVyZNmkT//v2d73nuued48MEH6dWrF1ar1bn/L3/5C2fOnCEiIoKoqCjnL4GpU6cSGRlZ7Q9FKzR9rlIqHPhWax1RyrF3gZNa65lKqabA70CU1rrMpbqrbfrc8uRnw1s9ISMRxrwHUeNcf08h6rj6OH3u3Llzy3xgWhtV+/S5SqlPgQ1AJ6XUcaXU3Uqpe5VSjtmweAEYoJTaCfwMPFFemNcod29j/VGAFS9AgXn9W0II4UrlPhTVWt9ezvFEYGS1VeQKUbfDhnfg1G74dTYMesjsioQQNWzKlClMmTLF7DJcqv59U7Q0FiuMLF5/9DVZf1QIUS81jEAHY/3RtldCXhr88k+zqxFCiGrXcAIdYOQLgIJNH8DpQ2ZXI4QQ1aphBXqz7kZ/ur0Alpc9yY8QQtQ1DSvQwVh/1M0TYpfAsU1mVyOEuITi6XOjoqLo0aMH69evL/P8+Ph4IiIuGlld6fPqooYX6AHNod80Y3vZX2T9USFqqeLpc7dv385LL73EU089ZXZJtV7DC3Qwhi16B8GxjcaMjEKIWi09Pd05sVVmZibDhw+nR48edO/ena+//tp5XlFREffccw/dunVj5MiR5OTkALBlyxaioqKIioo676v48fHxDB48mB49epz3r4BVq1YxdOhQbrnlFjp37sz48eMp/hLm888/T+/evYmIiGDq1KnO/UOHDqX4y5IpKSmEh4e7/HO5UHVMzlX3eAYY87x8/zj89Bx0HAVWm9lVCVErdZ/X3SXX3Tl5Z5nHi7/6n5ubS1JSEitWrADA09OTr776Cn9/f1JSUujXrx833ngjAPv37+fTTz/l/fff57bbbmPx4sVMmDCBO++8k1mzZjFkyBAee+wx5z1CQkL46aef8PT0ZP/+/dx+++3OUN66dSu7d+8mLCyMgQMHsm7dOgYNGsSMGTN49tlnAZg4cSLffvstN9xwgys+osvWMFvoYMzE2LgtnD4IW+aaXY0Q4gLFXS5xcXH88MMPTJo0Ca01WmuefvppIiMjGTFiBAkJCZw8eRKANm3aEB0dDUDPnj2Jj4/n7NmznD17liFDhgBGCBcrKCjgnnvuoXv37tx6663Exp6bartPnz60aNECi8VCdHQ08fHxAKxcuZK+ffvSvXt3VqxYwe7du2voEylfw2yhg2P90Znw+SRY9TJEjgPPUieJFKJBK68lXRP69+9PSkoKycnJLF26lOTkZLZs2YLNZiM8PNw5Za2Hh4fzPVar1dnlcimvv/46TZs2Zfv27djtdjw9PZ3HLrxWYWEhubm5TJs2jc2bN9OyZUtmzpzpvHdtmEK34bbQAbrcCC37QnYKrHvD7GqEEJcQFxdHUVERQUFBpKWlERISgs1mY+XKlRw5cqTM9wYGBhIYGOhczKJ42l2AtLQ0QkNDsVgsLFiwgKKiojKvVRzUwcHBZGZmnrdIdMkpdCu7eHRVNexAVwquesHY3vA2pCeaW48Qwqm4Dz06Oppx48Yxb948rFYr48ePZ/PmzXTv3p358+fTuXP5s3V/9NFHTJ8+nejoaErOMDtt2jTmzZtHVFQUcXFx5S50ERgYyD333ENERARXX301vXv3dh579NFHeffdd4mJiXGuL1rTKjR9rivU2PS5FbFoIuz5BqInwE1vl3++EPVcfZw+ty6q9ulzG4QRM8+tP3pil9nVCCFEpUigAwS1g153A9pYf1QIIeogCfRiVzwBHv5w8Gc4uMLsaoQwnVndscJQmc9fAr2YTxAMetjYXvYs2Mt+2i1Efebp6UlqaqqEukm01qSmpp43jLIiGu449NL0u8+YWvfkTtixCKLvMLsiIUzRokULjh8/TnJystmlNFienp60aNHist4jgV6SzcuYjXHJfbDib9BtjLFPiAbGZrPRpk0bs8sQl6nOdblk5Gfw0a6PyC7Ids0NIsdB0+6QngAb33HNPYQQwgXqXKA/+sujvLblNb7Y56JvYp23/ujrkGXOFwSEEOJy1blA/0OnPwAwP3Y+BUUFrrlJu2HQbjjkZ8j6o0KIOqPOBfoVLa+gXUA7Tmaf5LvD37nuRlc9DyjY/CGkHnTdfYQQoprUuUC3KAt3db8LgDm75mDXdtfcqFkERI8HeyEsn+maewghRDWqc4EOcE2ba2jm04zDaYdZdWyV62407Blw8zLmeTn6q+vuI4QQ1aDcQFdKzVFKnVJKXXKSE6XUUKXUNqXUbqXUL9Vb4sVsFhuTu04G4MOdH7ruyw/+YdB/urEt648KIWq5irTQ5wKjLnVQKRUIvAPcqLXuBtxaPaWV7eYONxPgEcCOlB1sPunCWRsHPgjewXD8N6OlLoQQtVS5ga61Xg2cLuOUO4AvtdZHHeefqqbayuRt8+aOzsY3OefsmuO6G3n6w9Anje3lM6Ew33X3EkKIKqiOPvSOQCOl1Cql1Bal1KRquGaF3N75drzcvFibsJa9p/e67kY9p0BQezh9CLZ85Lr7CCFEFVRHoLsBPYHrgKuBvyqlOpZ2olJqqlJqs1Jqc3XMEdHIsxFjO4wFXNxKt9qMOdPBWH80N8119xJCiEqqjkA/Dvyotc7SWqcAq4Go0k7UWr+nte6lte7VpEmTSt0st6CID9YcoshuPKCc1HUSbsqNH+J/4FjGsUr+FSqg8/XQsh/knIa1r7vuPkIIUUnVEehfA4OUUm5KKW+gL7CnGq5bqns/3sLfvtvDv340ulhCfUO5tu212LWdebvnueq2xvqjI/9mbG98F9KOu+5eQghRCRUZtvgpsAHopJQ6rpS6Wyl1r1LqXgCt9R7gB2AH8BvwgdbaZeu4TR3cFqtFMfuXg3y11QjVO7vdCcCSA0tIzUl11a2hZW/oehMU5hqzMQohRC1SkVEut2utQ7XWNq11C631h1rr2Vrr2SXO+ZfWuqvWOkJr/W9XFjygfTAzb+gKwBOLd7L16BnaN2rP0BZDySvKY+Geha68PYx4Diw22P4ZJO1w7b2EEOIy1Mlvik7sH86Efq3IL7QzdcEWktJyuLv73QB8tvczsgqyXHfzxm2h9x+R9UeFELVNnQx0gOdu6Eb/tkEkZ+Rxz/zNdArsTo+QHmTkZ7huat1iVzwOHgFwaCWsf8u19xJCiAqqs4Fus1p4Z3wPWjX2ZldCOo9+sZ27IoxJu+bvnk9+kQu/AOTdGK57xdhe9hf47X3X3UsIISqozgY6QCMfdz6Y3AtfDze+25HEtr2htA9sz6mcU3x76FvX3jzyNrjuVWN76aOw1cV990IIUY46HegAHZv68ebt0SgFry/fT+9A44tGH+36iCJ7kWtv3vuP54YyfjMDdi127f2EEKIMdT7QAYZ1bsqTozoDsGB5I4I9mxGfHs/KYytdf/MB98OVz4C2w5dTIc6Fi24IIUQZ6kWgA0wd0pabY5qTUwCZJwcCLp5at6Qhj8Ggh43FMP47BQ4sd/09hRDiAvUm0JVSvHhzd2JaBZKcFIVV+7IrdRebTmyqiZvD8Oeg771QlA+fTYD4ta6/rxBClFBvAh3A02blPxN7EurvT3ZKf8BopdcIpeDql6DHJCjMgU/GwbEa+GUihBAO9SrQAUL8PHl/Ui8sGQPRdnfWJ60nNjW2Zm5uscD1/4but0F+Jnw8FpK218y9hRANXr0LdICI5gG8OnYABWf6APCPDe/W3M0tVrjpXehyA+SlwYIxcMplc5UJIYRTvQx0gOsiQ5nQdRJaW/k95RfWHK7BULW6wdg50GEkZKfC/NGQerDm7i+EaJDqbaADPDOyH00tA0BpHv7xDc5m1+DycW7ucNt8aDMEMk/CvBvhzJGau78QosGp14FusSjevOYh0Ipcj1/50yerKCyy11wBNi/4w6fQsi+kH4f5N0J6Ys3dXwjRoNTrQAfo1qQjA0KHoCyFbE37H3/7rob7sz18Yfx/ITQazsQb3S+ZVV9+TwghLlTvAx1gWo97AHBvtJG5G+P45NejNVuAZwBM/ApCukHKPlhwE2SfrtkahBD1XoMI9KgmUfRu1htlzcU98Fee/XoXGw+5cGWj0ng3hklLIKgDnNxlDGnMTa/ZGoQQ9VqDCHTAObVuQLMNFOp87vt4C0dTs2u2CN8QmPQ1BLaGxN/hk9sg34WLcQghGpQGE+gDwwbSqVEncvVZunXcz5nsAv44fxMZuQU1W0hAc5j8Dfg3h6Mb4NPboSC3ZmsQQtRLDSbQlVLOZers/itp28SLfSczeXjRNorsNTCBV0mNwmHSN+ATAod/gc8nQWENDqkUQtRLDSbQAa5qfRXNfZtzPPMYd43MJMDLxvI9p3hl2d6aLya4vdH94tUY9v8IX/4Rigprvg4hRL3RoALdzeLGnd3uBOCb+I95+44YrBbFu6sOsmRrQs0X1LSrMfrFIwBiv4avp4G9BsfJCyHqlQYV6ACj24+msWdj9pzeg9XnAM/d0BWAxxfvYOvRMzVfUFg0TPgCbD6wYxF89zDUxBzuQoh6p8EFuqebJxO6TADgw10fMrFfa8b3bUV+oZ2pC7aQlJZT80W17AN3fAZunrBlLvzwlIS6EOKyNbhABxjXeRw+Nh9+TfqV2NRYZt7YjX5tG5OckcfU+VvIyXfxWqSlaTMExi0Eiw1+fRdWvFDzNQgh6rRyA10pNUcpdUoptauc83orpQqVUrdUX3mu4e/uz20dbwOMVrrNauGd8T1p2diLnQlpPPbF9ppZuu5CHUbArXNBWWHNq7D6XzVfgxCizqpIC30uMKqsE5RSVuAfwLJqqKlGTOg6AZvFxvIjy4lPi6exjzsfTu6Nj7uVb3ckMWvFAXMK63I93PweoGDF32DD2+bUIYSoc8oNdK31aqC8iUfuBxYDp6qjqJoQ4h3Cje1uRKOZu3suAB2b+vHm7TEoBa/+tI8fdp0wp7jut8CNbxnbPz4Nm2poGT0hRJ1W5T50pVRzYAxQg8sCVY8p3aagUHxz8BtOZRu/i4Z3acoTozoD8PCibcQmmjTfSo+JcO0rxvZ3f4Ztn5pThxCizqiOh6L/Bp7QWpc7gFopNVUptVkptTk52fwpZMMDwhnRegQF9gI+jv3Yuf9PQ9pyc0xzcgqKuGf+ZlIy88wpsM89cNXzxvbX02D3V+bUIYSoE6oj0HsBnyml4oFbgHeUUjeVdqLW+j2tdS+tda8mTZpUw62rrnjSrs/3fU56vtEaV0rx4s3diW4ZSMLZHO77eAt5hSaMfAEY+CAMfQq0HRb/EfZ+b04dQohar8qBrrVuo7UO11qHA18A07TWS6pcWQ2JCI6gb2hfsgqy+Hzv5879njYr703sSTN/TzbFn+GvS3aZM/IF4IonYMADYC805n05uMKcOoQQtVpFhi1+CmwAOimljiul7lZK3auUutf15dWM4lb6gtgF5Baem/kwxN+T9yf1wtNm4fPNx5mzLt6cApUyul563wNF+fDpHRC/zpxahBC1VkVGudyutQ7VWtu01i201h9qrWdrrWeXcu4UrfUXrinVdfqH9qdL4y6czj3NNwe/Oe9Y9xYBvHJrFAB//y6WX/aZ1PevFFzzT4iZAIU5xlzqxzebU4sQolZqkN8UvVDJqXU/2vURhfbzZz28PjKMB4a1x65hxie/c+BUphllgsUCN7wJEbdAfiZ8fDMk7TCnFiFErSOB7jCi1Qha+bXieOZxfjry00XHHxrRkVHdmpGRW8g98zeTll3DC2MUs1hhzGzofD3kphnrk56KM6cWIUStIoHuYLVYmRIxBYA5u+Zc9ADUYlG8Ni6KLqH+HE7JYvonv1NYZNJUt1Yb3DIH2o+A7FSYex3ELTWnFiFErSGBXsKN7W4k2CuYuNNxrE9cf9Fxb3c33p/Uk2Bfd9YeSOGFb2PNG/ni5gHjPoZ2wyE7BT67Hb6eLgtPC9GASaCX4GH1OG9q3dK0aOTN7Ak9sVkV8zYc4U8Ltpj3xSObF4z/Aq5+EawesPVjeHcgxK81px4hhKkk0C9wW6fb8LX5sunEJnYkl/7AsVd4Y966PQY/DzeWxZ7k6tdXmzfvi8UC/afDn1ZDaBSkHYW518OPz8ji00I0MBLoF/Bz92Ncp3GA0Zd+KaMiQvnh4SEMaBdEalY+9368hT8v2kZajkkPS0M6wx9/Nr6EpHXrlvQAACAASURBVCywYRb8ZwgkbjWnHiFEjZNAL8WErhNwt7iz4ugKDqUduuR5zQO9+Pjuvsy8oSsebha+3JrAqH+vZs1+k8aqW21w5dPwx58gqAOk7IUPRsCqf0CRSb9ohBA1RgK9FMFewYxuP9qYWnfX3DLPtVgUUwa2YemDg4luGUhSWi4TP/yNZ7/eRXZ+YZnvdZnmPeHeNdD3PmO6gFUvwocjIXmfOfUIIWqEBPolTOk2BYuy8L9D/+NEVvn94+2a+PLFvf157OpO2KyK+RuOcO0ba9hypLyp5F3E5gXXvAyTvgH/FpD4O/xnMGycDXaThlsKIVxKAv0SWvm34qrWV1FoL2RB7IIKvcfNamH6le1ZMn0gnZv5EZ+aza2zN/CPH+LMm62x7RUwbT1E3QGFufDDE7BgNJw9Zk49QgiXkUAvQ/GkXV/s+4K0vLQKv69bWABfzxjIfUPbAfDuqoOMnrXOvMUyPANgzLvGItTewXB4Nbw7ALZ9AmaNoxdCVDsJ9DJ0DerKgLABZBdm81ncZ5f1Xg83K0+M6sx/7+1PeJA3cScyGP32Wt5eecC8b5h2uR6mbYRO10FeOiy5DxZNgEzzFxsRQlSdBHo5ilvpC/csJKcw57Lf37N1Y5Y+OJhJ/VtTUKT51497ufU/GziUbNIEX75N4A8L4aZ3wcMf4r6Fd/pB3Hfm1COEqDYS6OXo06wPEUERnMk7w5IDlVu3w9vdjedHR7Dg7j408/dk69GzXPvmGuauO4zdbkKXh1IQfQfctw7CBzumDrgDlkwzJvwSQtRJEujlKDm17txdcymwV3489+AOTfjx4SHcHNOc3AI7M/8Xy4QPfyXh7OW3/KtFYCtjFMyol8HNE7YtNKYOOLzanHqEEFUigV4BV7a8knD/cBKzEvkx/scqXSvAy8Zr46KZPaEnQT7urD+YyqjXV/PfzcfMmejLYoF+98Gf1kBYDKQdg3k3wA9PQYFJv2iEEJUigV4BVouVOyPuBEqfWrcyRkU048eHhzCya1My8gp57Isd3DN/C8kZJk301aQj3P0TDH0alBU2vmNMHZCwxZx6hBCXTQK9gq5vez0hXiHsP7OfNQlrquWawb4e/GdiT169NQo/DzeW7znJ1f9ezfc7k6rl+pfNaoOhT8Afl0NwJ0jZBx9cBStfkqkDhKgDJNAryN3qzsSuEwH4cGfpU+tWhlKKsT1b8OPDQxjUPpjTWfnct/B3Hvpsq3mrIjXvAX/6BfpNB22HX1425oRJ3mtOPUKICpFAvwy3dLwFP3c/fj/1O9tObavWa4cFejH/rj48P7obnjYLS7YlcvW/V5u3KLXNC0a9CJP/BwEtIWkbzB4MG96WqQOEqKUk0C+Dr7svf+j0B+DSC2BUhcWimNQ/nO8fHEKPVoGcSM9l8pzfeOarnWTlmTTRV5vBcN96iJ4ARXnw49Mw/0Y4e9SceoQQlySBfpnGdxmPh9WDVcdWceDMAZfco02wD/+9dwCPjzIm+lr461GufXMNm+NNmujL0x9uehv+8Cn4NIH4NfDOAGOFJJk6QIhaQwL9MgV5BXFT+5sA+Gj3Ry67j9WimDa0Pd/MGETnZn4cSc3m1v9s4KXv95BbYNJEX52vNaYO6Hw95GcYa5h+dgdknjKnHiHEeSTQK2FKtylYlZWlh5ay74xr5xjvEurPNzMGMf3KdijgP78c4sZZa9mVYNI3On2CjcWpx/zHmDpg71Jj6oA9/zOnHiGEkwR6JbTwa8Ho9qMp1IVM/3k6p7Jd20J1d7Pw2NWd+eK+AbQJ9mHfyUxuensdb/2835yJvpSCqD/AtA3Q5grITjUm+Vo0EU7srPl6hBBABQJdKTVHKXVKKbXrEsfHK6V2KKV2KqXWK6Wiqr/M2uepPk8R3SSaE1knmP7zdLIKslx+zx6tGrH0gcFMGRBOoV3z6k/7GPvueg6cMmmir4AWMHEJXPMvcPOCPd/A7EGw8DY49ps5NQnRgKnyvvWolBoCZALztdYRpRwfAOzRWp9RSl0DzNRa9y3vxr169dKbN2+uZNm1w5ncM0z8fiJH0o8wMGwgbw1/C5vFViP3Xncghcf+u53EtFxsVsWEfq25f1gHGvu418j9L5KWYCxMvfkjKJ6VMnwwDP4ztL3SaNULIapMKbVFa92r1GMV+Rq7Uioc+La0QL/gvEbALq118/KuWR8CHeBY+jEmfD+B07mnubnDzczsPxNVQ+GVnlvA37/dw+dbjqE1+Hm4cd+V7bhrYBs8bdYaqeEiWSmw8V347X0oXhQkrAcMfgQ6XWvMHSOEqLSaDPRHgc5a6z9e4vhUYCpAq1ateh45cqTce9cFO5J3cPePd5NblMuM6Bn8KepPNXr/2MR0Xv4hjtWOLyGFBnjy56s6cnOPFlgtJrWMc9Ng04fGF5GyU4x9TTrDoD9DxFiwuplTlxB1XI0EulLqSuAdYJDWOrW8a9aXFnqxFUdX8NDKh9Bo/j7o79zY7sYar2HN/mReWhpHbJKx1F3nZn48cU1nhnZsUmP/arhIfjZsXQDr3oT048a+wNYw6CFjnVObpzl1CVFHuTzQlVKRwFfANVrrCo3jq2+BDvDJnk946beXcFNuvHvVu/QL7VfjNdjtmm+2J/KvH/c651kf0C6Ip67pQvcWATVej1NhPuxYBGtfh9MHjX2+zWDADOh5J3j4mlebEHWISwNdKdUKWAFM0lqvr2hR9THQAV7Z9ArzYufha/Nl3jXz6Niooyl15BYUsWDDEWatPEBajjHJ1+joMB4d2YmWjb1NqQkAexHEfg1rXoOTjiGOXo2g733Q5x7wbmxebULUAVUKdKXUp8BQIBg4CTwH2AC01rOVUh8AY4HiDvHCS92spPoa6HZt57FfHmPZkWU09W7KwmsX0tSnqWn1pGUX8M6qA3y0Pp78QjvuVgsT+7dmxpXtaWTWiBgwpgzYvwxWvwLHHUMc3X2h113Qfwb4mfeZCVGbVbmF7gr1NdAB8oryuGfZPWw9tZWOjToyb9Q8fN3N7VI4fiab15bt46ttCcaIGE83pl/ZnikDws0bEQNGsB9ZB2tehYMrjH1WD+gxEQY8AI1am1ebELWQBLoJzuaeZeL3E4lPj6d/aH/eHvF2jY1RL8vuxDRe/j6ONfuNkSdhAZ78eWQnxsQ0N29ETLGELUZXTNy3xmtlhcjbYNDD0KSTubUJUUtIoJvkWMYxJiw1xqjf1P4mnh/wvHmjTS6wel8yL30fx54SI2KeurYLQzoEm1/jqT3Gw9OdX4AuAhR0ucH4klJYjLm1CWEyCXQT7UzeyV0/3kVuUS7ToqZxX/R9ZpfkZLdrlmxL4JUf95KYlgvAoPbBPHlNZyKamzgiptjpw7DuDdi2EIryjX3thsOQR6H1AHNrE8IkEugmW3l0JQ+tegi7tvPCwBec0+/WFrkFRczfEM+sFQdIzzUW0rgpOoxHr+5Ei0Ymjogplp50blqB4jlzWvU3vn3afoRMKyAaFAn0WuDTuE958dcXcVNuvD3ibQaE1b4W5tnsfN5eeYB564+QX2SMiJk8oDXTr2xPoLeJI2KKZZ+GX2cbf3Id0wo0izSCvcsNYDHx4a4QNUQCvZZ4bfNrfLT7I3xsPswbNY9OjWvng75jp7N57ad9fLU1AQB/TzdmDGvPpP4mj4gplpcBm+fA+lmQ5Zi6OKiD8fA08jawmv/wWQhXkUCvJezazhOrn+CH+B8I8Q5h4bULaebTzOyyLmlXQhovfb+HdQeMmRyaB3rxyMiO3BTdHIvZI2IACnKMZfDWvQlpjjVOA1pC77sh8g/gH2pufUK4gAR6LZJXlMfUZVP5/dTvdGjUgXmj5uHn7md2WZektWb1/hReWrqHuBMZAHQN9eepazszuEMTk6tzKCowRsSsfQ1SHDNPKIvRvx49HjpdA24e5tYoRDWRQK9l0vLSmLB0AvHp8fQL7cc7w9/BVsu7CYrsmiVbE3h12bkRMYM7GCNiuoXVghExAHY77P/RaLXv+wHsxgNevBpB91sh+g4IjZaHqKJOk0CvhY5nHGfC0gmk5qZyY7sb+dvAv5k//rsCcguKmLs+nrdXHiAjtxClYEx0c/48smPtGBFTLCsFdv4Xti48N2cMQEg3iBkP3W8D31ryLwwhLoMEei21O2U3d/54JzmFOdwbdS/To6ebXVKFncnKZ9bKAyzY4BgR42bh1p4tGNuzBTEtA2vXL6ek7bDtE9jxOeScNvZZ3KDD1UarvePV8iBV1BkS6LXYL8d+4YGVD2DXdp4f8DxjOowxu6TLcux0Nq8s28vX2xKd+8KDvLkppjljYprTOsjHxOouUJhndMVs+wT2/+T4FirgHQyR44xwb1bmGi5CmE4CvZb7fO/nvLDxBazKytvD32Zg84Fml3TZ9p7I4L+bj/H19kSSM/Kc+3u0CmRMjxZc3z3U3NkdL5Rxwmixb1sIyXHn9odGGQ9Su98qU/mKWkkCvQ54fcvrzNk1B283b+ZdM4/OjTubXVKlFNk16w6ksGRrAj/sPkF2vtEKtlkVQzuFcHNMc67sHFI7xrODMdtj4u9GX/uuL859YcnqboyOiR5vTDcgS+aJWkICvQ6waztPrnmS7w9/TxOvJiy8diGhvnV7HHVWXiHLYk/w1dZE1u5Pxu74T83f043rIkO5Kbo5vcMb144x7QAFubD3OyPcD64AHAX7NjW6ZGImyKyPwnQS6HVEflE+f/rpT2w+uZn2ge2Zd808/N39zS6rWpxKz+Wb7Yl8tTWB3Ynpzv3NA70YE9Ocm2Ka0z6kFi1Dl5YAOz4zwr14yTyA5j2NVnvEWPAKNK8+0WBJoNchaXlpTPp+EofSDtGnWR9mj5hd68eoX659JzP4amsCX29NcI5pB4hsEcCYmObcEBVGsG8t+SKQ1nDsN9j2Mez6CvKNL1dh9YAu1xvh3naozCMjaowEeh2TkJnAhKUTSMlJ4fq21/PioBdr1zDAamK3azYeTmXJ1gS+33mCjDzji0BWi2JIh2DG9GjBVV2a4uVeS8IyPwv2fGuE++HV5/b7N4eoPxjhHtTOvPpEgyCBXgftTt3NnT8YY9SnRk7l/pj7zS7JpXILili+5yRf/Z7AL/uSKXR0uPt6uDEqohljYprTr22Q+asqFTt7FLZ9aoySOXvk3P6W/YwvLnW9CTzrR3eZqF0k0Ouo1cdXc/+K+7FrOzP7z2Rsx7Fml1QjUjPz+HZHEl9uTWD7sbPO/c38PRkdE8aYmOZ0blZLwtJuh6Prjb722CVQkG3sd/OCNkOg3TBoPxyC2suUA6JaSKDXYf/d91+e3/A8VmXlrWFvMbjFYLNLqlGHkjNZsjWBr7YlcOx0jnN/l1B/xsSEMTq6OU39PU2ssIS8DIj92gj3o+vPPxbQEtpdaQR8mytkjLuoNAn0Ou6N39/gg50f4OXmxdxRc+ka1NXskmqc1potR87w5dYEvtuRRFpOAQAWBQPbB3NTdHNGRTTDx6OWjBdPT4SDK43hj4dWQnbquWPKAmE9jHBvNwxa9JKpB0SFSaDXcVprnlzzJEsPLyXYK5iF1y4kzDfM7LJMk1dYxMq4ZL7aepwVcacoKDL+G/ayWRnZrSkD2wfTNdSfDk198XCrBQ9U7XY4sQMO/myE/NGNYC84d9zD39E942jBN25rXq2i1pNArwfyi/K5d/m9bDqxiXYB7Zh3zTwCPGrJtLUmOpudz3c7k1iyNYFN8WfOO+ZmUbRr4kuXUD+6hvnTJdT4Y/qQyLxMOLIODvxstOBT959/vFGbc633NoPBU/53FudIoNcTaXlpTP5+MgfTDtK7WW9mj5iNu7UWzY9isqOp2SzdlcTOhDT2JKVzOCWL0v7zbuLnQdfQ4oD3o2uoP22CfXCzWmq+aDBGzBxcabTgD606N/0AgLJCyz7nAj4sRsa8N3BVCnSl1BzgeuCU1vqiqeiUMUD6DeBaIBuYorX+vbyiJNArJzEzkfFLx5OSk8K1ba7l5cEv18sx6tUhO7+QvScy2JOUwZ6kdOefLMf8MiV5uFno1MyPLs0cIR8WQOdQP/w9a7hv214EiVuNlvuBn+H4pnOzQgJ4BhpfZCoO+MCWNVufMF1VA30IkAnMv0SgXwvcjxHofYE3tNZ9yytKAr3y9qTuYfIPk8kpzOGe7vfwQI8HzC6pzrDbNcfOZLMnKZ3YpAxiE42QTzibU+r5LRp5lWjN+9M11J+Wjb1q7pdobhocXmME/MEVcObw+ceDOhjDItsNg9YDwaMWTZ8gXKLKXS5KqXDg20sE+n+AVVrrTx2v9wJDtdZJZV1TAr1q1hxfw/0r7qdIF/GXvn9hXOdxZpdUp6XlFBCXlE6ssyWfwd6TGeQX2i8618/Djc6hfueFfKdmfjUzg+TpQ45wXwmHfjk3FQGAxQat+jkerg6HZpFgMakbSbiMqwP9W+BlrfVax+ufgSe01heltVJqKjAVoFWrVj2PHDly4SniMizet5iZG2YCcFXrq3is12N1fobG2qSwyM6hlCxHa94I+djEdFIy8y4616KgTbCPEfCOB7Dtgn0JC/R0Xd98UQEkbDn3cDXxd9AlfgF5Bxlj3lv0NvreQyPBvRYtOCIqpdYEeknSQq8en8V9xmtbXiOnMAcvNy+mRk5lctfJ9W5Cr9okOSPP2R9f3KI/mJxFkf3i/y/ZrIqWjb0JD/IhPMiHNsHehAcb22GBXtU7lUH2aWOOmYM/w4EVkH78/OPKAk26GOHePMYYC9+0G7jVkonQRIVIl0s9dyLrBK9sfoUf438EINw/nKf7Pk3/sP4mV9Zw5BYUceBUJrGJRsjvPZHB4ZQsTqTnXvI97lYLrYK8Lwr68GAfQv09qzZPvNaQegDi1xot94StcCr2/AesYCzk0bSbEe5hMdC8BwR3kgU9ajFXB/p1wAzOPRR9U2vdp7xrSqBXv/WJ63np15eIT48H4Orwq3m016M082lmbmENWE5+EUdOZxGfksXhlGzjZ6rx+lTGxV03xTzcLLR2hr2PM+zbBPvQ1N+jcg9lC3LgxE5I+N0YSZP4O6Tsx7mQRzGbt7EUX5ijFd+8hzE2XvrjAePBelZ+IZl5hWTmFpLh+Hnx6wIy8wrJKHGs+HWXUD8+mNy7Uvev6iiXT4GhQDBwEngOsAForWc7hi3OAkZhDFu8s7zuFpBAd5X8onzmx87nvR3vObth7ou6jwldJ2CzSDdMbZKVV0h8ahbxKdnEp2ZxOCWLI6lG8JfWT1/M02ZxduGEBzta946wb+J3mWGfmw5J2x2t+N+Nn2ePXnyeRwCERRvhXhz0AS3qzIRjWmvyi+xk5xWdH7J5BaUG7nmv8wrJzC1w7itt2Ovl6hbmz3cPVG5eJvliUQOUlJnEPzf9k+VHlwPQLqAdT/d9mj6h5f7jSdQCGbkFHEnN5nBK1nmt+vjUbE5n5V/yfT7uVlo7W/XeNA/0xs2qsCqF1aKwWIq3weLY5/yjjOMe+WfwTd2Jb+oOfFJ24pWyHVv2qYvuVeQdTEHTaIqaRVPULAYdFoPyCzl3L+dP4/y8QrvjTxF5BSW2C+2O10Xnzikouuzz84vKvk518nG34uvphq+HG76eNvw8ireNn36epb22Obf9PN0I9K7clwIl0BuwtQlreenXlziaYbS6rmlzDY/2epQQ7xCTKxOVlZZT4Aj381v38alZnM0uKP8CldCU00RaDhFpOUSUOkh3y2EaqcyLzkvQQeywt2WHvR07dBt22tuSTu0YWWOzKjxtVvw9beeFra+n2yUC2VZqQPu4u5k6L78EegOXV5TH3F1zeX/n++QV5eFj82Fa1DRu73K7dMPUM2ez853hfjglmxNpORTaNXa7pkgb/b9Fdk2RLt5nvLYX/7Rz0b7zjmuMfUV2mukTdCo6QGf7AbrqA3ThED5c/BD4kL0Zu3U4B2lJvLUVx6ytSXYLw83dhoebFQ83i/HHVmLbzYqHzYK71YKHzVLuec5tNwue551/7jq1ZjHyKpJAF4CxtN0/fvsHK4+tBKB9YHue6fsMvZqV+t+GEJfHXmSMrCnui0/cCkk7oKiU5wFWdwjuCE06GUMpQzobPxuFywibckigi/OsPr6al359ieOZxjjl69tezyO9HiHYK9jkykS9U1RgDJc8sQuS98CpOEjeC2mlPHgFY/Ht4A7QpPO5kG/SGRq3kUnJHCTQxUVyC3P5aNdHfLDzA/Lt+fjafJkRM4NxncbhZpEWknCxvAxI3ucI+T2QXBz0x0o/3+phtOhDOhsB36QzhDha9A0s6CXQxSUdSz/GS7+9xJqENQB0atSJZ/o9Q0xIjMmViQYpNx1S9p0L+VN7jKC/8Fuvxdw8HS36Lkb3TYijRV+Pg14CXZRJa82qY6t4+beXScxKBGB0u9E83PNhgryCTK5OCIygT95botvG8Sc9ofTzSwZ9cddNSGcIbF3ng14CXVRITmEOH+78kDm75lBgL8DP3Y/7Y+7nto63Ya3j/ycQ9VRumhH0xS354sDPSCz9fKsHBLUzwj6og/GzeNvTv2ZrryQJdHFZjqQf4aVfX2Jd4joAujTuwjP9niGqSZTJlQlRQTlnHQEfV6LrJg4yyphiyrep0U8f1N4R9I7twFa1qlUvgS4um9aaFUdX8PKmlzmRdQKAmzvczIM9HqSxZ2OTqxOiknLTjTVcUw44fjr+nD4IhZeYSM3qYSzcXbI1H9wRgtubst6rBLqotOyCbN7f+T5zd8+l0F6Iv7s/D/Z4kLEdxko3jKg/7HZjhE3K/hJBv88YV19Wq94n5Fy4lwx6F/bVS6CLKjucdpgXf32RjUkbAegW1I2/9PsLEcEXTcApRP2Sl2EEe3FrvjjwUw+U0ap3h8btSgR9iS4cr8AqlSOBLqqF1pplR5bxz03/5FT2KRSKWzrewgMxDxDoWbX/SIWoc+x2YzjlhUGfsv/SD2UBfJoY67/eNq9St5VAF9UquyCb2Ttms2D3Agp1IYEegTzU4yHGdBiDRcmc2UKQl3muVZ9aMvAPQGEOtBkCk/9XqUtLoAuXOHT2EH//9e/8duI3AFr5tWJE6xEMbzWciOAICXchLmS3G2PnC3KgScdKXUICXbiM1pof4n/glc2vcKrEnNkh3iEMazmMEa1H0LNpT5lOQIhqIoEuXK7QXsjvJ3/n56M/8/PRnzmZfdJ5LMAjgKEthjK81XD6h/XH083TxEqFqNsk0EWN0lqzO3U3Px/9meVHljvXOAXwcvNiUPNBDG81nCEthuDn7mdeoULUQRLowlSHzh5i+dHl/Hz0Z2JTY5373Sxu9A3ty4hWI7iy5ZUyb4wQFSCBLmqNxMxEVhxdwfKjy9l6ait2baz1qFDEhMQwvNVwhrceTnPf5iZXKkTtJIEuaqXTuadZdWwVy48sZ2PSRgrs59bD7NK4C8NaDWNEqxG0C2x3eSvZC1GPSaCLWi8zP5M1CWv4+ejPrD6+mpzCHOex1v6tjZa7DIcUQgJd1C15RXlsTNzI8qPLWXVsFWfzzjqPyXBI0dBJoIs6q9BeyNZTW1l+ZLkMhxQCCXRRT1R0OGTPpj1p6t1U+t1FvVTlQFdKjQLeAKzAB1rrly843gqYBwQ6znlSa720rGtKoIuqOnT2kBHuR5efNxwSwM/djw6BHejQqAMdG3WkQ6MOtA9sL+PeRZ1XpUBXSlmBfcBVwHFgE3C71jq2xDnvAVu11u8qpboCS7XW4WVdVwJdVKfi4ZC/HP+FuNNx5/W7lxTqE3ou5B2BHx4Qjs1iq+GKhaicsgK9Ik+U+gAHtNaHHBf7DBgNlGwSaaB4Qb4AoIy5I4WofmG+YUzoOoEJXSegtSYlJ4X9Z/az/+x+9p3Zx/4z+zl49iBJWUkkZSWx+vhq53vdLG60CWhDh8BzrfmOjTpKt42ocyrSQr8FGKW1/qPj9USgr9Z6RolzQoFlQCPABxihtd5SyrWmAlMBWrVq1fPIkSPV9fcQolyF9kKOZhw1gv7MuaA/nnm81POl20bURlXtcqlIoP/Zca1XlVL9gQ+BCK0dXwMshXS5iNoiuyCbA2cPnAv5s0bgS7eNqI2q2uWSALQs8bqFY19JdwOjALTWG5RSnkAwcAohajlvmzeRTSKJbBLp3FfcbVPcii8O+cvptgnxDqGxZ2MaeTaSsBc1oiItdDeMh6LDMYJ8E3CH1np3iXO+BxZprecqpboAPwPNdRkXlxa6qIsK7YUcTT/KvrP7nF03ZXXbFAvwCKCxZ2OCPIOMn14X/PQMMo55NcbbzVv67sUlVcewxWuBf2MMSZyjtf67Uup5YLPW+hvHyJb3AV+MB6SPa62XlXVNCXRRn2QVZDm7bfaf2c+htEOk5KRwOvc0Z/POOichqwhPq2epoe/8heB17hdDoEcgVhetLl+faK3Jt+dTUFRAgb2AQnshBfYLtosKKNSF5Z5zyWMXXusS5xQUFdC+UXteGPhCpf4uVe1ywTGmfOkF+54tsR0LDKxUdULUAz42H6KaRBHVJOqiY0X2Is7mnSU1N5XTuac5nXPauZ2a49jn2E7NTSW3KJfErEQSs8ofLGZRFgI9Ai9u7XsZLX5/D3+syopFWVAolFJYsKDUBdsoLMpinFfi9XnbJd5vURZQOLedxxzXKK6teLs4UPOLSvxxvC6wF5BXlOfcf95rRwgXvy6wF5BflG+8dhy71HUvvEdt4qp/gclEGEK4mNViNQK2AvO9a63JLsx2hv6FwX/eL4DcVNLy0pyvD5w9UAN/m7rLzeKGu8Udm9WGzWLDzeKGzVLGttWGm3Jznl/qeZdxrZKvfW2+rvk7uuSqQohKUUrhY/PBx+ZDS/+W5Z5fYC/gTO6ZS7b20/PSsWNHa+38ed42Grt2HNd2NPq87fP2lfH+4i6l4u3zro1GoXC3uuNudcdmseFh9TBeOwLWw+rh3Ha3up//2uLufG/xe0rdLvn6gv02i61BzNIpFEJcQQAAA3hJREFUgS5EHWaz2AjxDiHEO8TsUkQtUP9/ZQkhRAMhgS6EEPWEBLoQQtQTEuhCCFFPSKALIUQ9IYEuhBD1hAS6EELUExLoQghRT5i2SLRSKhmo6ytcBAMpZhdRi8jncT75PM6Rz+J8Vfk8Wmutm5R2wLRArw+UUpsvNetZQySfx/nk8zhHPovzuerzkC4XIYSoJyTQhRCinpBAr5r3zC6glpHP43zyeZwjn8X5XPJ5SB+6EELUE9JCF0KIekICXQgh6gkJ9EpQSrVUSq1USsUqpXYrpR40uyazKaWsSqmtSqlvza7FbEqpQKXUF0qpOKXUHvX/7dzNi01xHMfx9yejGEpsxMxiLEST8pDFMGXhoYSwtCALS3lKiX9As5BY2YynMo0FipWIhY0sSMgsLIhhmCl5yAb5WJyjplneOc33Ovf7qtv53bM459Ot87nnd865V1oTnSmSpCPlcfJC0qCkGdGZppKkC5JGJb0Yt26epLuSXpXLuVXsKwu9Mb+Bo7a7gR5gv6Tu4EzRDgFD0SGaxFngtu2lwHJa+HOR1AEcBFbbXgZMA3bFpppyl4DNE9YdB+7ZXgzcK99PWhZ6A2yP2H5Sjr9THLAdsaniSOoEtgL90VmiSZoDrAPOA9j+aftLbKpwbcBMSW1AO/AhOM+Usv0A+Dxh9Q7gcjm+DOysYl9Z6JMkqQtYCTyKTRLqDHAM+BMdpAksAsaAi+UlqH5Js6JDRbH9HjgFvAVGgK+278SmagrzbY+U44/A/Co2moU+CZJmA9eBw7a/ReeJIGkbMGr7cXSWJtEGrALO2V4J/KCi6fT/qLw2vIPii24hMEvS7thUzcXFs+OVPD+ehd4gSdMpynzA9o3oPIF6ge2S3gBXgfWSrsRGCjUMDNv+N2O7RlHwrWoj8Nr2mO1fwA1gbXCmZvBJ0gKAcjlaxUaz0BsgSRTXSIdsn47OE8n2Cdudtrsobnbdt92yZ2C2PwLvJC0pV20AXgZGivYW6JHUXh43G2jhm8Tj3AL2luO9wM0qNpqF3pheYA/F2ejT8rUlOlRqGgeAAUnPgBXAyeA8YcqZyjXgCfCconNa6m8AJA0CD4ElkoYl7QP6gE2SXlHMYvoq2Vf+9D+llOohz9BTSqkmstBTSqkmstBTSqkmstBTSqkmstBTSqkmstBTSqkmstBTSqkm/gKjgLOp6LpzhwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uBeMYwpMOPyI",
        "colab_type": "text"
      },
      "source": [
        "# Translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ad9Fn5HrOO6i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def translate(sentence, encoder, decoder):\n",
        "  attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
        "\n",
        "  sentence = preprocess_sentence(sentence)\n",
        "\n",
        "  inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
        "  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
        "                                                         maxlen=max_length_inp,\n",
        "                                                         padding='post')\n",
        "  inputs = tf.convert_to_tensor(inputs)\n",
        "\n",
        "  result = ''\n",
        "\n",
        "  hidden = [tf.zeros((1, units))]\n",
        "  enc_out, enc_hidden = encoder(inputs, hidden)\n",
        "\n",
        "  dec_hidden = enc_hidden\n",
        "  dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
        "\n",
        "  for t in range(max_length_targ):\n",
        "    predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
        "                                                         dec_hidden,\n",
        "                                                         enc_out)\n",
        "\n",
        "    predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "\n",
        "    result += targ_lang.index_word[predicted_id] + ' '\n",
        "\n",
        "    # until the predicted word is <end>.\n",
        "    if targ_lang.index_word[predicted_id] == '<end>':\n",
        "      return result, sentence\n",
        "\n",
        "    # the predicted ID is fed back into the model, no teacher forcing.\n",
        "    dec_input = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "  return result, sentence"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmPB_EoAWxKP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "b6ef32d4-74b3-49af-fa4c-3a8b516c8cdd"
      },
      "source": [
        "result, sentence = translate(u'esta es mi vida.', encoder_bah, decoder_bah)\n",
        "print('Input: %s' % (sentence))\n",
        "print('Predicted translation: {}'.format(result))"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: <start> esta es mi vida . <end>\n",
            "Predicted translation: this is my life . <end> \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kZWVIi421_4o",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "29d2c572-b503-4cf4-af5a-d44834aa3fcb"
      },
      "source": [
        "result, sentence = translate(u'esta es mi vida.', encoder_dp, decoder_dp)\n",
        "print('Input: %s' % (sentence))\n",
        "print('Predicted translation: {}'.format(result))"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: <start> esta es mi vida . <end>\n",
            "Predicted translation: this is my life . <end> \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xrB48Fwk7XW8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "13fbe38e-ddf1-4343-9934-ad82be435850"
      },
      "source": [
        "result, sentence = translate(u'¿todavia estan en casa?', encoder_bah, decoder_bah)\n",
        "print('Input: %s' % (sentence))\n",
        "print('Predicted translation: {}'.format(result))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: <start> ¿ todavia estan en casa ? <end>\n",
            "Predicted translation: are you at home ? <end> \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iLZcz8TbR09_",
        "colab_type": "text"
      },
      "source": [
        "# Next Steps\n",
        "\n",
        "\n",
        "*   Training on larger dataset\n",
        "*   Model tuning\n",
        "*   Try out other attention scores such as multiplicative\n",
        "*   Train on other seq2seq tasks\n",
        "\n",
        "\n",
        "\n"
      ]
    }
  ]
}