EncoderDecoderArchitecture_English_to_Hindi.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "EncoderDecoderArchitecture_English_to_Hindi.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/C3Suryansu/English-Hindi-Transliteration/blob/master/EncoderDecoderArchitecture_English_to_Hindi.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1GqTjeV47m4B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torch.autograd import Variable\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "\n",
        "# Instantiates the device to be used as GPU/CPU based on availability\n",
        "device_gpu = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Visualization tools\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from IPython.display import clear_output\n",
        "\n",
        "import random\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KpuvHS0mxwCd",
        "colab_type": "text"
      },
      "source": [
        "## Data Management"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eYrAa5laSptM",
        "colab_type": "text"
      },
      "source": [
        "### Alphabets Setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-a04ZKx7Sh-J",
        "colab_type": "code",
        "outputId": "ea021934-8e68-42cf-dc88-27fbf266b8ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "eng_alphabets = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'\n",
        "pad_char = '-PAD-'\n",
        "\n",
        "eng_alpha2index = {pad_char: 0}\n",
        "for index, alpha in enumerate(eng_alphabets):\n",
        "    eng_alpha2index[alpha] = index+1\n",
        "\n",
        "print(eng_alpha2index)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'-PAD-': 0, 'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7, 'H': 8, 'I': 9, 'J': 10, 'K': 11, 'L': 12, 'M': 13, 'N': 14, 'O': 15, 'P': 16, 'Q': 17, 'R': 18, 'S': 19, 'T': 20, 'U': 21, 'V': 22, 'W': 23, 'X': 24, 'Y': 25, 'Z': 26}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cPSZsy1kXd9w",
        "colab_type": "code",
        "outputId": "547c8a74-d9a0-4864-a033-f29e6619da74",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "# Hindi Unicode Hex Range is 2304:2432. Source: https://en.wikipedia.org/wiki/Devanagari_(Unicode_block)\n",
        "\n",
        "hindi_alphabets = [chr(alpha) for alpha in range(2304, 2432)]\n",
        "hindi_alphabet_size = len(hindi_alphabets)\n",
        "\n",
        "hindi_alpha2index = {pad_char: 0}\n",
        "for index, alpha in enumerate(hindi_alphabets):\n",
        "    hindi_alpha2index[alpha] = index+1\n",
        "\n",
        "print(hindi_alpha2index)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'-PAD-': 0, 'ऀ': 1, 'ँ': 2, 'ं': 3, 'ः': 4, 'ऄ': 5, 'अ': 6, 'आ': 7, 'इ': 8, 'ई': 9, 'उ': 10, 'ऊ': 11, 'ऋ': 12, 'ऌ': 13, 'ऍ': 14, 'ऎ': 15, 'ए': 16, 'ऐ': 17, 'ऑ': 18, 'ऒ': 19, 'ओ': 20, 'औ': 21, 'क': 22, 'ख': 23, 'ग': 24, 'घ': 25, 'ङ': 26, 'च': 27, 'छ': 28, 'ज': 29, 'झ': 30, 'ञ': 31, 'ट': 32, 'ठ': 33, 'ड': 34, 'ढ': 35, 'ण': 36, 'त': 37, 'थ': 38, 'द': 39, 'ध': 40, 'न': 41, 'ऩ': 42, 'प': 43, 'फ': 44, 'ब': 45, 'भ': 46, 'म': 47, 'य': 48, 'र': 49, 'ऱ': 50, 'ल': 51, 'ळ': 52, 'ऴ': 53, 'व': 54, 'श': 55, 'ष': 56, 'स': 57, 'ह': 58, 'ऺ': 59, 'ऻ': 60, '़': 61, 'ऽ': 62, 'ा': 63, 'ि': 64, 'ी': 65, 'ु': 66, 'ू': 67, 'ृ': 68, 'ॄ': 69, 'ॅ': 70, 'ॆ': 71, 'े': 72, 'ै': 73, 'ॉ': 74, 'ॊ': 75, 'ो': 76, 'ौ': 77, '्': 78, 'ॎ': 79, 'ॏ': 80, 'ॐ': 81, '॑': 82, '॒': 83, '॓': 84, '॔': 85, 'ॕ': 86, 'ॖ': 87, 'ॗ': 88, 'क़': 89, 'ख़': 90, 'ग़': 91, 'ज़': 92, 'ड़': 93, 'ढ़': 94, 'फ़': 95, 'य़': 96, 'ॠ': 97, 'ॡ': 98, 'ॢ': 99, 'ॣ': 100, '।': 101, '॥': 102, '०': 103, '१': 104, '२': 105, '३': 106, '४': 107, '५': 108, '६': 109, '७': 110, '८': 111, '९': 112, '॰': 113, 'ॱ': 114, 'ॲ': 115, 'ॳ': 116, 'ॴ': 117, 'ॵ': 118, 'ॶ': 119, 'ॷ': 120, 'ॸ': 121, 'ॹ': 122, 'ॺ': 123, 'ॻ': 124, 'ॼ': 125, 'ॽ': 126, 'ॾ': 127, 'ॿ': 128}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSw1SMZmx9A3",
        "colab_type": "text"
      },
      "source": [
        "### Helper functions for data pre-processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OcS6ByndOxrC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "non_eng_letters_regex = re.compile('[^a-zA-Z ]')\n",
        "\n",
        "# Remove all English non-letters\n",
        "def cleanEnglishVocab(line):\n",
        "    line = line.replace('-', ' ').replace(',', ' ').upper()\n",
        "    line = non_eng_letters_regex.sub('', line)\n",
        "    return line.split()\n",
        "\n",
        "# Remove all Hindi non-letters\n",
        "def cleanHindiVocab(line):\n",
        "    line = line.replace('-', ' ').replace(',', ' ')\n",
        "    cleaned_line = ''\n",
        "    for char in line:\n",
        "        if char in hindi_alpha2index or char == ' ':\n",
        "            cleaned_line += char\n",
        "    return cleaned_line.split()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ob3F9Dh4PChB",
        "colab_type": "text"
      },
      "source": [
        "### Dataset Loading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KGSeoMGg0FTy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import Dataset\n",
        "import xml.etree.ElementTree as ET\n",
        "\n",
        "class TransliterationDataLoader(Dataset):\n",
        "    def __init__(self, filename):\n",
        "        self.eng_words, self.hindi_words = self.readXmlDataset(filename, cleanHindiVocab)\n",
        "        self.shuffle_indices = list(range(len(self.eng_words)))\n",
        "        random.shuffle(self.shuffle_indices)\n",
        "        self.shuffle_start_index = 0\n",
        "        \n",
        "    def __len__(self):\n",
        "        return len(self.eng_words)\n",
        "    \n",
        "    def __getitem__(self, idx):\n",
        "        return self.eng_words[idx], self.hindi_words[idx]\n",
        "    \n",
        "    def readXmlDataset(self, filename, lang_vocab_cleaner):\n",
        "        transliterationCorpus = ET.parse(filename).getroot()\n",
        "        lang1_words = []\n",
        "        lang2_words = []\n",
        "\n",
        "        for line in transliterationCorpus:\n",
        "            wordlist1 = cleanEnglishVocab(line[0].text)\n",
        "            wordlist2 = lang_vocab_cleaner(line[1].text)\n",
        "\n",
        "            # Skip noisy data\n",
        "            if len(wordlist1) != len(wordlist2):\n",
        "                print('Skipping: ', line[0].text, ' - ', line[1].text)\n",
        "                continue\n",
        "\n",
        "            for word in wordlist1: \n",
        "                lang1_words.append(word)\n",
        "            for word in wordlist2:\n",
        "                lang2_words.append(word)\n",
        "\n",
        "        return lang1_words, lang2_words\n",
        "    \n",
        "    def get_random_sample(self):\n",
        "        return self.__getitem__(np.random.randint(len(self.eng_words)))\n",
        "    \n",
        "    def get_batch_from_array(self, batch_size, array):\n",
        "        end = self.shuffle_start_index + batch_size\n",
        "        batch = []\n",
        "        if end >= len(self.eng_words):\n",
        "            batch = [array[i] for i in self.shuffle_indices[0:end%len(self.eng_words)]]\n",
        "            end = len(self.eng_words)\n",
        "        return batch + [array[i] for i in self.shuffle_indices[self.shuffle_start_index : end]]\n",
        "    \n",
        "    def get_batch(self, batch_size, postprocess = True):\n",
        "        eng_batch = self.get_batch_from_array(batch_size, self.eng_words)\n",
        "        hindi_batch = self.get_batch_from_array(batch_size, self.hindi_words)\n",
        "        self.shuffle_start_index += batch_size + 1\n",
        "        \n",
        "        # Reshuffle if 1 epoch is complete\n",
        "        if self.shuffle_start_index >= len(self.eng_words):\n",
        "            random.shuffle(self.shuffle_indices)\n",
        "            self.shuffle_start_index = 0\n",
        "            \n",
        "        return eng_batch, hindi_batch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-FCCi-SerZS-",
        "colab_type": "code",
        "outputId": "508dfe7d-1416-4ed5-d46b-30dd0af971a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        }
      },
      "source": [
        "train_data = TransliterationDataLoader('NEWS2012-Training-EnHi-13937.xml')\n",
        "test_data = TransliterationDataLoader('NEWS2012-Ref-EnHi-1000.xml')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Skipping:  BARHARWA JUNCTION  -  बरहरवा\n",
            "Skipping:  STATE BNK TR  -  स्टेट बैंक ऑफ त्रावणकोर\n",
            "Skipping:  SOUTH ARLINGTON CHURCH OF CHRIST  -  साउथ अर्लिंग्टन\n",
            "Skipping:  KING EDWARD VII  -  किंग एडवर्ड\n",
            "Skipping:  DIBANG VALLEY  -  दिबंगवैली\n",
            "Skipping:  ORDER OF VASA  -  ऑडर ऑफ़ द वासा\n",
            "Skipping:  AZAMNAGAR ROAD  -  आज़मनगर\n",
            "Skipping:  CAPE TOWN  -  केपटाउन\n",
            "Skipping:  NEW ZEALAND  -  न्यूज़ीलैंड\n",
            "Skipping:  SEA OF THE HEBRIDES  -  सी ऑफ हरब्रिड्‍स\n",
            "Skipping:  RAMCOIND  -  राम्को इंड\n",
            "Skipping:  KELVINGROVE ART GALLERY AND MUSEUM  -  केल्व‍िनग्रोव आर्ट एण्ड म्युज़ियम\n",
            "Skipping:  AUSTRALIAN NATIONAL UNIVERSITY  -  ऑस्ट्रेलियननेशनल यूनिवर्सिटी\n",
            "Skipping:  JAHAN AARA  -  जहाँआरा\n",
            "Skipping:  NAVABHARAT FERRO ALLOYS  -  नव भारत फ़ैरो अलॉय\n",
            "Skipping:  RAMA LINGESHWARA  -  रामालिंगेश्वर\n",
            "Skipping:  FAKHRUN NISA  -  फखरुन्निसा\n",
            "Skipping:  REDIFF.COM INDIA LIMITED  -  रेडिफ़ डॉट कॉम इंडिया लिमिटेड\n",
            "Skipping:  OMKARNATH THAKUR  -  ओंकार नाथ ठाकुर\n",
            "Skipping:  OPENTV  -  ओपन टीवी\n",
            "Skipping:  ENVOY COMMUNICATIONS GROUP  -  एन्वॉय कम्युनिकेशंस\n",
            "Skipping:  WAR OF THE HOLY LEAGUE  -  वार ऑफ होली लीग\n",
            "Skipping:  VAPARAISO CHURCH OF CHRIST  -  व्हापरासिओ\n",
            "Skipping:  PARIS CHARLES DE GAULLE  -  पेरिस रॉसे चार्ल्स डे ग्यूले\n",
            "Skipping:  PARKWAY APOSTOLIC  -  पार्क वे अपोस्टोलिक\n",
            "Skipping:  MAUNA LOA  -  मौनालोआ\n",
            "Skipping:  MASS MUTUAL LIFE  -  मास म्युच्युअल लाइफ़ इंश्योरेंस\n",
            "Skipping:  STATS CHIPPAC  -  स्टेट्सचिपपैक\n",
            "Skipping:  NEWFOUNDLAND  -  न्यू फाउंडलैंड\n",
            "Skipping:  LONDONHEATHROW  -  लंदन हीथ्रो\n",
            "Skipping:  RETALIX  -  रेटालिक्स लि.\n",
            "Skipping:  SRISAILAM  -  श्री शैलम\n",
            "Skipping:  KARA-KUM  -  काराकुम\n",
            "Skipping:  WIND RIVER  -  विंडरिवर\n",
            "Skipping:  NETAJI SUBHASH CHANDRA BOSE  -  नेताजी सुभाषचंद्र बोस\n",
            "Skipping:  ROCKBROOK UNITED  -  रॉकब्रुक यूनाइटेड मेथोडिस्ट\n",
            "Skipping:  WALTER SCOTT  -  वॉल्टरस्कॉट\n",
            "Skipping:  COLOURPLUS FASHIONS  -  कलर प्लस फ़ैशन्स\n",
            "Skipping:  BAL KRISHNA  -  बालकृष्णा\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7l-iaCVdx5Ez",
        "colab_type": "text"
      },
      "source": [
        "### Basic Data Visualization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XIwPqcJH05hK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def infer(net, word, max_output_chars, device = 'cpu'):\n",
        "  net.eval().to(device)\n",
        "  word_ohe = word_rep(word,eng_alpha2index)\n",
        "  output = net(word_ohe, max_output_chars)\n",
        "  return output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IjY06ghEx76b",
        "colab_type": "code",
        "outputId": "29644ae8-be57-4b33-8ae0-3ce09fff9b42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "print(\"Train Set Size:\\t\", len(train_data))\n",
        "print(\"Test Set Size:\\t\", len(test_data))\n",
        "\n",
        "print('\\nSample data from train-set:')\n",
        "for i in range(10):\n",
        "    eng, hindi = train_data.get_random_sample()\n",
        "    print(eng + ' - ' + hindi)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Set Size:\t 20543\n",
            "Test Set Size:\t 1000\n",
            "\n",
            "Sample data from train-set:\n",
            "HAATH - हाथ\n",
            "KAL - कल\n",
            "KANDHARI - कंधारी\n",
            "TRACK - ट्रैक\n",
            "SOHAIL - सोहेल\n",
            "JALANGI - जलंगी\n",
            "KAMYAB - कामयाब\n",
            "GENERAL - जनरल\n",
            "JOHN - जॉन\n",
            "PANDURANGA - पांडुरंग\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KpDP1_KYZIkv",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "### Encoding the words"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JE3at5C7Sy5F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def word_rep(word, letter2index, device = 'cpu'):\n",
        "    rep = torch.zeros(len(word)+1, 1, len(letter2index)).to(device)\n",
        "    for letter_index, letter in enumerate(word):\n",
        "        pos = letter2index[letter]\n",
        "        rep[letter_index][0][pos] = 1\n",
        "    pad_pos = letter2index[pad_char]\n",
        "    rep[letter_index+1][0][pad_pos] = 1\n",
        "    return rep\n",
        "\n",
        "def gt_rep(word, letter2index, device = 'cpu'):\n",
        "    gt_rep = torch.zeros([len(word)+1, 1], dtype=torch.long).to(device)\n",
        "    for letter_index, letter in enumerate(word):\n",
        "        pos = letter2index[letter]\n",
        "        gt_rep[letter_index][0] = pos\n",
        "    gt_rep[letter_index+1][0] = letter2index[pad_char]\n",
        "    return gt_rep"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-yE3jToOrfzP",
        "colab_type": "code",
        "outputId": "3f29e08f-000e-45ed-c1e3-5f84d5430c99",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "source": [
        "eng, hindi = train_data.get_random_sample()\n",
        "eng_rep = word_rep(eng, eng_alpha2index)\n",
        "print(eng, eng_rep)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MONA tensor([[[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]],\n",
            "\n",
            "        [[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]],\n",
            "\n",
            "        [[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]],\n",
            "\n",
            "        [[0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]],\n",
            "\n",
            "        [[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "          0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uMcDjIberhc3",
        "colab_type": "code",
        "outputId": "2cde8e4c-5ced-4042-9bf7-adecd9333559",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "hindi_gt = gt_rep(hindi, hindi_alpha2index)\n",
        "print(hindi, hindi_gt)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "मोना tensor([[47],\n",
            "        [76],\n",
            "        [41],\n",
            "        [63],\n",
            "        [ 0]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GrC3tSnm4rUk",
        "colab_type": "text"
      },
      "source": [
        "## Network Architecture"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D4OgdZ_DVVC5",
        "colab_type": "text"
      },
      "source": [
        "### Encoder-Decoder (using GRU)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6w8ffT3w4lkK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_OUTPUT_CHARS = 30\n",
        "class Transliteration_EncoderDecoder(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_size, hidden_size, output_size, verbose=False):\n",
        "        super(Transliteration_EncoderDecoder, self).__init__()\n",
        "        \n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        \n",
        "        self.encoder_rnn_cell = nn.GRU(input_size, hidden_size)\n",
        "        self.decoder_rnn_cell = nn.GRU(output_size, hidden_size)\n",
        "        \n",
        "        self.h2o = nn.Linear(hidden_size, output_size)\n",
        "        self.softmax = nn.LogSoftmax(dim=2)\n",
        "        \n",
        "        self.verbose = verbose\n",
        "        \n",
        "    def forward(self, input, max_output_chars = MAX_OUTPUT_CHARS, device = 'cpu', ground_truth = None):\n",
        "        \n",
        "        # encoder\n",
        "        out, hidden = self.encoder_rnn_cell(input)\n",
        "        \n",
        "        if self.verbose:\n",
        "            print('Encoder input', input.shape)\n",
        "            print('Encoder output', out.shape)\n",
        "            print('Encoder hidden', hidden.shape)\n",
        "        \n",
        "        # decoder\n",
        "        decoder_state = hidden\n",
        "        decoder_input = torch.zeros(1, 1, self.output_size).to(device)\n",
        "        outputs = []\n",
        "        \n",
        "        if self.verbose:\n",
        "            print('Decoder state', decoder_state.shape)\n",
        "            print('Decoder input', decoder_input.shape)\n",
        "        \n",
        "        for i in range(max_output_chars):\n",
        "            \n",
        "            out, decoder_state = self.decoder_rnn_cell(decoder_input, decoder_state)\n",
        "            \n",
        "            if self.verbose:\n",
        "                print('Decoder intermediate output', out.shape)\n",
        "            \n",
        "            out = self.h2o(decoder_state)\n",
        "            out = self.softmax(out)\n",
        "            outputs.append(out.view(1, -1))\n",
        "            \n",
        "            if self.verbose:\n",
        "                print('Decoder output', out.shape)\n",
        "                self.verbose = False\n",
        "            \n",
        "            max_idx = torch.argmax(out, 2, keepdim=True)\n",
        "            if not ground_truth is None:\n",
        "                max_idx = ground_truth[i].reshape(1, 1, 1)\n",
        "            one_hot = torch.FloatTensor(out.shape).to(device)\n",
        "            one_hot.zero_()\n",
        "            one_hot.scatter_(2, max_idx, 1)\n",
        "            \n",
        "            decoder_input = one_hot.detach()\n",
        "            \n",
        "        return outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cra9toTiOoPm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = Transliteration_EncoderDecoder(len(eng_alpha2index), 256, len(hindi_alpha2index), verbose=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v4zaJq2pOrM8",
        "colab_type": "code",
        "outputId": "c5831adc-6eb6-4202-a73a-dd1a9a8f4ac9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "out = infer(net, 'INDIA', 30)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Encoder input torch.Size([6, 1, 27])\n",
            "Encoder output torch.Size([6, 1, 256])\n",
            "Encoder hidden torch.Size([1, 1, 256])\n",
            "Decoder state torch.Size([1, 1, 256])\n",
            "Decoder input torch.Size([1, 1, 129])\n",
            "Decoder intermediate output torch.Size([1, 1, 256])\n",
            "Decoder output torch.Size([1, 1, 129])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_pdzBmQOsjO",
        "colab_type": "code",
        "outputId": "7a84f3df-e8ec-40fc-8b10-96bb10141976",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "print(len(out))\n",
        "for i in range(len(out)):\n",
        "    print(out[i].shape, list(hindi_alpha2index.keys())[list(hindi_alpha2index.values()).index(torch.argmax(out[i]))])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30\n",
            "torch.Size([1, 129]) ँ\n",
            "torch.Size([1, 129]) ॴ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ड\n",
            "torch.Size([1, 129]) ॢ\n",
            "torch.Size([1, 129]) ॢ\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NEg49N9e7oTY",
        "colab_type": "text"
      },
      "source": [
        "### Encoder-Decoder with Attention \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8z-1QDAz8F_d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Transliteration_EncoderDecoder_Attention(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_size, hidden_size, output_size, verbose=False):\n",
        "        super(Transliteration_EncoderDecoder_Attention, self).__init__()\n",
        "        \n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        \n",
        "        self.encoder_rnn_cell = nn.GRU(input_size, hidden_size)\n",
        "        self.decoder_rnn_cell = nn.GRU(hidden_size*2, hidden_size)\n",
        "        \n",
        "        self.h2o = nn.Linear(hidden_size, output_size)\n",
        "        self.softmax = nn.LogSoftmax(dim=2)\n",
        "        \n",
        "        self.U = nn.Linear(self.hidden_size, self.hidden_size)\n",
        "        self.W = nn.Linear(self.hidden_size, self.hidden_size)\n",
        "        self.attn = nn.Linear(self.hidden_size, 1)\n",
        "        self.out2hidden = nn.Linear(self.output_size, self.hidden_size)   \n",
        "        \n",
        "        self.verbose = verbose\n",
        "        \n",
        "    def forward(self, input, max_output_chars = MAX_OUTPUT_CHARS, device = 'cpu', ground_truth = None):\n",
        "        \n",
        "        # encoder\n",
        "        encoder_outputs, hidden = self.encoder_rnn_cell(input)\n",
        "        encoder_outputs = encoder_outputs.view(-1, self.hidden_size)\n",
        "        \n",
        "        if self.verbose:\n",
        "            print('Encoder output', encoder_outputs.shape)\n",
        "        \n",
        "        # decoder\n",
        "        decoder_state = hidden\n",
        "        decoder_input = torch.zeros(1, 1, self.output_size).to(device)\n",
        "        \n",
        "        outputs = []\n",
        "        U = self.U(encoder_outputs)\n",
        "        \n",
        "        if self.verbose:\n",
        "            print('Decoder state', decoder_state.shape)\n",
        "            print('Decoder intermediate input', decoder_input.shape)\n",
        "            print('U * Encoder output', U.shape)\n",
        "        \n",
        "        for i in range(max_output_chars):\n",
        "            \n",
        "            W = self.W(decoder_state.view(1, -1).repeat(encoder_outputs.shape[0], 1))\n",
        "            V = self.attn(torch.tanh(U + W))\n",
        "            attn_weights = F.softmax(V.view(1, -1), dim = 1) \n",
        "            \n",
        "            if self.verbose:\n",
        "                print('W * Decoder state', W.shape)\n",
        "                print('V', V.shape)\n",
        "                print('Attn', attn_weights.shape)\n",
        "            \n",
        "            attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
        "                                 encoder_outputs.unsqueeze(0))\n",
        "            \n",
        "            embedding = self.out2hidden(decoder_input)\n",
        "            decoder_input = torch.cat((embedding[0], attn_applied[0]), 1).unsqueeze(0)\n",
        "            \n",
        "            if self.verbose:\n",
        "                print('Attn LC', attn_applied.shape)\n",
        "                print('Decoder input', decoder_input.shape)\n",
        "                \n",
        "            out, decoder_state = self.decoder_rnn_cell(decoder_input, decoder_state)\n",
        "            \n",
        "            if self.verbose:\n",
        "                print('Decoder intermediate output', out.shape)\n",
        "                \n",
        "            out = self.h2o(decoder_state)\n",
        "            out = self.softmax(out)\n",
        "            outputs.append(out.view(1, -1))\n",
        "            \n",
        "            if self.verbose:\n",
        "                print('Decoder output', out.shape)\n",
        "                self.verbose = False\n",
        "            \n",
        "            max_idx = torch.argmax(out, 2, keepdim=True)\n",
        "            if not ground_truth is None:\n",
        "                max_idx = ground_truth[i].reshape(1, 1, 1)\n",
        "            one_hot = torch.zeros(out.shape, device=device)\n",
        "            one_hot.scatter_(2, max_idx, 1) \n",
        "            \n",
        "            decoder_input = one_hot.detach()\n",
        "            \n",
        "        return outputs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PMD3zjdJO0Oj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net_attn = Transliteration_EncoderDecoder_Attention(len(eng_alpha2index), 256, len(hindi_alpha2index), verbose=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YoiQwbntO5UH",
        "colab_type": "code",
        "outputId": "21a68f71-0411-4b2f-df1d-6206093ec6ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "out = infer(net_attn, 'INDIA', 30)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Encoder output torch.Size([6, 256])\n",
            "Decoder state torch.Size([1, 1, 256])\n",
            "Decoder intermediate input torch.Size([1, 1, 129])\n",
            "U * Encoder output torch.Size([6, 256])\n",
            "W * Decoder state torch.Size([6, 256])\n",
            "V torch.Size([6, 1])\n",
            "Attn torch.Size([1, 6])\n",
            "Attn LC torch.Size([1, 1, 256])\n",
            "Decoder input torch.Size([1, 1, 512])\n",
            "Decoder intermediate output torch.Size([1, 1, 256])\n",
            "Decoder output torch.Size([1, 1, 129])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K9WSPgzlO6k8",
        "colab_type": "code",
        "outputId": "0069e37f-36da-47ea-8265-310824eaeae2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 544
        }
      },
      "source": [
        "print(len(out))\n",
        "for i in range(len(out)):\n",
        "    print(out[i].shape, list(hindi_alpha2index.keys())[list(hindi_alpha2index.values()).index(torch.argmax(out[i]))])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "30\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n",
            "torch.Size([1, 129]) ऀ\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cyE2tSnmAW6x",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H893cimDtTUE",
        "colab_type": "text"
      },
      "source": [
        "### Core Trainer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m804jsH7AXSV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_batch(net, opt, criterion, batch_size, device = 'cpu', teacher_force = False):\n",
        "    \n",
        "    net.train().to(device)\n",
        "    opt.zero_grad()\n",
        "    eng_batch, hindi_batch = train_data.get_batch(batch_size)\n",
        "    \n",
        "    total_loss = 0\n",
        "    for i in range(batch_size):\n",
        "        \n",
        "        input = word_rep(eng_batch[i], eng_alpha2index, device)\n",
        "        gt = gt_rep(hindi_batch[i], hindi_alpha2index, device)\n",
        "        outputs = net(input, gt.shape[0], device, ground_truth = gt if teacher_force else None)\n",
        "        \n",
        "        for index, output in enumerate(outputs):\n",
        "            loss = criterion(output, gt[index]) / batch_size\n",
        "            loss.backward(retain_graph = True)\n",
        "            total_loss += loss\n",
        "        \n",
        "    opt.step()\n",
        "    return total_loss/batch_size"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p-eZaBxstWz9",
        "colab_type": "text"
      },
      "source": [
        "### Training Helper"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rjto129ssrpr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_setup(net, lr = 0.01, n_batches = 100, batch_size = 10, momentum = 0.9, display_freq=5, device = 'cpu'):\n",
        "    \n",
        "    net = net.to(device)\n",
        "    criterion = nn.NLLLoss(ignore_index = -1)\n",
        "    opt = optim.Adam(net.parameters(), lr=lr)\n",
        "    teacher_force_upto = n_batches//n_batches\n",
        "    \n",
        "    loss_arr = np.zeros(n_batches + 1)\n",
        "    \n",
        "    for i in range(n_batches):\n",
        "        loss_arr[i+1] = (loss_arr[i]*i + train_batch(net, opt, criterion, batch_size, device = device, teacher_force = i<teacher_force_upto ))/(i + 1)\n",
        "        \n",
        "        if i%display_freq == display_freq-1:\n",
        "            clear_output(wait=True)\n",
        "            \n",
        "            print('Iteration', i, 'Loss', loss_arr[i])\n",
        "            plt.figure()\n",
        "            plt.plot(loss_arr[1:i], '-*')\n",
        "            plt.xlabel('Iteration')\n",
        "            plt.ylabel('Loss')\n",
        "            plt.show()\n",
        "            print('\\n\\n')\n",
        "            \n",
        "    torch.save(net, 'model.pt')\n",
        "    return loss_arr"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TZY6RvqLtdX8",
        "colab_type": "text"
      },
      "source": [
        "### Training without Attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1oQ3ZIWvtjfN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = Transliteration_EncoderDecoder(len(eng_alpha2index), 256, len(hindi_alpha2index))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E6LjVKQfoVMU",
        "colab_type": "code",
        "outputId": "4997efca-f6af-46ba-ecae-c6d4ab62c9a5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 469
        }
      },
      "source": [
        "%%time\n",
        "train_setup(net, lr=0.001, n_batches=2000, batch_size = 64, display_freq=10, device = device_gpu)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration 1999 Loss 0.1783284842967987\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAfGUlEQVR4nO3df5RcZZ3n8fcn3SRRSBAk4UcIdNAEBxwE7MnoSITIr0DcxNGzigiDqx7AIYMO7kIwHHcWYQnM0bM4sCssy+ioGB1dnOyGTAQFJK6QNBh+JExICBES+dEShh8T6KST7/5Rtzq3K9XVVdV1q6qrPq9z+nTd595b9e1b3fXt53nu8zyKCMzMzAqNaXQAZmbWnJwgzMysKCcIMzMrygnCzMyKcoIwM7OiOhsdQK0cdNBB0dXV1egwzMxGlYcffvgPETGp2L6WSRBdXV309PQ0Ogwzs1FF0u+G2ucmJjMzK8oJwszMinKCMDOzopwgzMysKCcIMzMrygkCeOm1t/jkLb/hpdffanQoZmZNwwkC+NYvNrB68za+dc+GRodiZtY0WmYcRDWOvmo5ff27B7a//9CzfP+hZxnXOYb115zVwMjMzBqvrWsQD1w+mznvPWRge/w+Y5h//GE8cMXsBkZlZtYc2jpBTJ44ngnjcpWozjGir383E8Z1MnnC+AZHZmbWeG3dxATwyvYdAJz/gSPZuTvodUe1mRngBMG3z3s/7160nAP2Hculp05vdDhmZk2jrZuYADrGCICbfrmBdc+/2uBozMyaR9snCCmXIHbsCi75/iMNjsbMrHm0dRNT18Jlg7afeXn7QNnmxXMbEZKZWdNo6xrEPh2qqNzMrJ20dYIQxRNBvtnJzKydtXWCWHnFbMYU5IIxypWbmbW7tk4Qs264l90xuGx3wKzr721MQGZmTaStE8QDl8/mkP3HDapFjO+Up9owMyPjBCFpjqT1kjZKWlhk/2cl9Upak3x9IbXvAkkbkq8Lsohv8sTxvPRa36BaxFv9wcxrf8HRVy3P4iXNzEaNzG5zldQB3AycDmwBVktaGhHrCg79UUQsKDj3QOA/A91AAA8n575S6zgLm5jy0rO8mpm1oyxrEDOBjRGxKSJ2AEuA+WWeeyZwd0RsS5LC3cCcLIL0ra5mZsVlmSCmAM+ltrckZYU+IekxST+RNLWScyVdKKlHUk9vb29VQf76io9QmAs6BL9e+JGqns/MrFU0upP6/wBdEXEcuVrCdys5OSJujYjuiOieNGlSVQFMnjiefToHX4Z9Osd4ym8za3tZJoitwNTU9uFJ2YCIeDki+pLN24D3l3tuLZ08Y9LAuhDTJ+/HyTOqSzZmZq0kywSxGpguaZqkscA5wNL0AZIOTW3OA55MHq8AzpB0gKQDgDOSspo7+qrlrFj7Iq/39QOw4aU3WLH2Rd/FZGZtL7MEERH9wAJyH+xPAj+OiLWSrpY0LznsUklrJT0KXAp8Njl3G/B1cklmNXB1UlZzD1w+m3nHHzYwFmJcp7zsqJkZGc/mGhF3AXcVlH0t9fhK4Mohzr0duD3L+GDPsqP52137+sPLjpqZ0fhO6oY7+qrl/OChZweVff+hZ93EZGZtr+0TRL6JqTO513V85xg3MZmZ4QQx0MS0a1eujamvf7ebmMzMcIIA4A9v9NHddQAApx9zML1v9A1zhplZ63OCAG45v5u3j+0AoGOMuOX87gZHZGbWeG29JjXkOqnTE/Mtf+IFuhYuY1znGNZfc1YDIzMza6y2r0HkO6nHJtNtjO1wJ7WZGThBDHRS70xqETt27aZTcie1mbW9tk8QkOuknve+wwa2V23OZNC2mdmo0vZ9EAD3re8d1A/x3Ctvuh/CzNqeaxDk+iHOfu8hA9vj93E/hJmZEwS5foj9377PwPZbOz1YzszMCSKx7d92DDyePnk/D5Yzs7bnBMGeNSHyvCaEmZkTBLBnLESe14QwM3OCAPaMhcjzmhBmZk4QgNeEMDMrxgmCPU1MyaqjbmIyM8MJAtjTxJSsOuomJjMznCAANzGZmRWTaYKQNEfSekkbJS0scdwnJIWk7mS7S9KbktYkX9/OMs58E1Oy6qhHUpuZkeFcTJI6gJuB04EtwGpJSyNiXcFxE4AvAQ8VPMXTEXF8VvGlDSw7mrQxeSS1mVm2NYiZwMaI2BQRO4AlwPwix30duB54K8NYhvWHN/p4RzLdhkdSm5llmyCmAM+ltrckZQMknQhMjYhlRc6fJum3ku6XNKvYC0i6UFKPpJ7e3t6qA82PpP7X7TsBj6Q2M4MGdlJLGgN8E/hKkd3PA0dExAnAZcAdkiYWHhQRt0ZEd0R0T5o0qepYCvsgxnX4NlczsywTxFZgamr78KQsbwLwXuA+SZuBDwBLJXVHRF9EvAwQEQ8DTwMzsgq0sA+ib5dvczUzyzJBrAamS5omaSxwDrA0vzMiXo2IgyKiKyK6gAeBeRHRI2lS0smNpKOA6cCmrAL1ba5mZnvLLEFERD+wAFgBPAn8OCLWSrpa0rxhTv8w8JikNcBPgIsjIrN1QPNNTGM79lyOrne+3U1MZtbWFBHDHzUKdHd3R09PT9XnH3XlMnYXuRRedtTMWpmkhyOiu9g+j6ROfHj6QUx5x9sGlbmj2szamRNE4jebtrH1X98cVPZPa37PrOvvbVBEZmaN5QSReODy2RwycdygskP3H+8ahJm1LSeIxKwb7uWF1waPnn7+1bdcgzCztuUEkXjg8tkcsv+eGsQYuQZhZu3NCSIxeeJ4Tn3PwQPbuwNOfc9kD5Yzs7blBJHwYDkzs8GcIBL5wXJ5HfJtrmbW3jJbD2K0mXXDvfT17x7Y3hW521z/+YkXPFDOzNqSaxCJwk5qcCe1mbU3J4jErBvu5YVXfZurmVmeE0RiqCmpWmOmKjOzyjlBJFZeMZuud759UNl+4zpY6SYmM2tTThCJyRPHs/nl7YPK3ujbxcxrf+FbXc2sLTlBpJwy4yCOONAzupqZgRPEIL/ZtI1nt3lGVzMzcIIYxB3VZmZ7OEGkFOuo7nrn291RbWZtyQkiZdYN9+7VUb355e1uYjKztpRpgpA0R9J6SRslLSxx3CckhaTuVNmVyXnrJZ2ZZZx5bmIyM9sjs7mYJHUANwOnA1uA1ZKWRsS6guMmAF8CHkqVHQOcAxwLHAbcI2lGROzKKl4zMxssyxrETGBjRGyKiB3AEmB+keO+DlwPvJUqmw8siYi+iHgG2Jg8X6ZWXjGbt+3TMahsTFJuZtZuskwQU4DnUttbkrIBkk4EpkbEskrPTc6/UFKPpJ7e3t4RBzx54nje3Dm4krIbPFjOzNpSwzqpJY0Bvgl8pdrniIhbI6I7IronTZpUk7jGaIjXqsmzm5mNHlmuB7EVmJraPjwpy5sAvBe4TxLAIcBSSfPKODcznWPGsGPX7uEPNDNrcVnWIFYD0yVNkzSWXKfz0vzOiHg1Ig6KiK6I6AIeBOZFRE9y3DmSxkmaBkwHVmUYq5mZFcisBhER/ZIWACuADuD2iFgr6WqgJyKWljh3raQfA+uAfuAS38FkZlZfmS45GhF3AXcVlH1tiGNPKdi+Frg2s+DMzKwkj6Qu047+3b6TyczaihNEgVJjHnwnk5m1EyeIApMnjm90CGZmTcEJogI7+n37q5m1DyeIIvbpKD5abqhyM7NW5ARRxK+v+EjR8p27gq6FhbOCmJm1JieIIkr1Q7gWYWbtwgmiQjt3+V4mM2sPThBDcD+EmbU7J4ghiOKJYOeu8IA5M2sLThBD8IA5M2t3ThBDKNVR7fEQZtYOnCCq5GYmM2t1ZSUISfsmK8AhaYakeZL2yTa0xrvr0pOG3NfnWoSZtbhyaxC/AsZLmgL8HDgf+E5WQTWLYw7bv9EhmJk1TLkJQhGxHfg48N8j4t8Dx2YX1ujgUdVm1srKThCSPgh8Bsh/KnZkE1JzWfXVUxsdgplZQ5SbIL4MXAncmSwHehRwb3ZhNQ9P/21m7aqsBBER90fEvIi4Pums/kNEXJpxbE3jg0cdOOQ+NzOZWasq9y6mOyRNlLQv8ASwTtJ/KuO8OZLWS9ooaWGR/RdLelzSGkkrJR2TlHdJejMpXyPp25X+YLX0wws/WHK/k4SZtaJym5iOiYjXgI8By4Fp5O5kGpKkDuBm4CzgGODT+QSQckdE/HFEHA/cAHwzte/piDg++bq4zDgzM3nCuEaHYGZWV+UmiH2ScQ8fA5ZGxE6Gn3FiJrAxIjZFxA5gCTA/fUCSdPL2LeM5G2bVotNK7nctwsxaTbkJ4hZgM7kP8V9JOhJ4reQZMAV4LrW9JSkbRNIlkp4mV4NI92tMk/RbSfdLmlVmnJnyHU1m1k7K7aT+VkRMiYizI+d3wNCz2VUgIm6OiHcBVwBXJcXPA0dExAnAZcAdkiYWnivpQkk9knp6e3trEU5Jw93R5FqEmbWScjup95f0zfyHsaRvkKtNlLIVmJraPjwpG8oSck1YRERfRLycPH4YeBqYUXhCRNwaEd0R0T1p0qRyfpQRO/PYg0vud5Iws1ZRbhPT7cDrwCeTr9eAvx/mnNXAdEnTJI0FzgGWpg+QND21ORfYkJRPSjq5ScZcTAc2lRlrpm45v3vYY5wkzKwVdJZ53Lsi4hOp7f8iaU2pEyKiX9ICYAW5Ude3J4PsrgZ6ImIpsEDSacBO4BXgguT0DwNXS9oJ7AYujoht5f9Y2dq8eK6TgJm1vHITxJuSToqIlQCSPgS8OdxJEXEXcFdB2ddSj780xHk/BX5aZmwNMXnCOF56vW/I/V0Ll7F58dw6RmRmVlvlNjFdDNwsabOkzcBNwEWZRTUKrFp0GhpmeWrXMsxsNCv3LqZHI+J9wHHAccndRR/JNLJR4Jnrhq8hOEmY2WhV0YpyEfFaanDbZRnEM+qU04zUtXAZ655/tQ7RmJnVzkiWHB2mgaV9lJMkzr5xJXc8tDn7YMzMamQkCaJpp8VohHLmavrqnWvd5GRmo0bJBCHpdUmvFfl6HTisTjGOCqsWnVb2hH5ucjKz0aBkgoiICRExscjXhIgo9xbZtrFq0Wll39p69o0rueX+DRlHZGZWvZE0MdkQhpuOI++65U/RtXAZ//exUjOQmJk1hiJaoyuhu7s7enp6Gh3GIJX0N4zrEHcu+BDHHLp/hhGZmQ0m6eGIKDqHkGsQGdq8eG7ZtYm+XeFmJzNrKk4QGbvl/O6ykwS42cnMmocTRB3ccn43mxfPHXZqjrQFd6xxbcLMGsoJoo6eua78JidwbcLMGsud1A0y46rl7OjfXdE5N517PB89bq9VW83Mqlaqk9oJosEqHVkt4HtfmMlJ767PCnpm1tqcIEYBJwozawTf5joKbF48l7Gd5b8dAZx32yqOXnSXp+0ws0y4BtGEqpnQzzUKM6uGaxCjzObFc8ue+C8vX6PwXU9mViuuQTS5mdfeU3Lt66GM7RA/89QdZjaMhtUgJM2RtF7SRkkLi+y/WNLjktZIWinpmNS+K5Pz1ks6M8s4m1l+hthKBtkB7Eim7pi2cBkrN/ZmE5yZtbTMahCSOoCngNOBLcBq4NMRsS51zMT8EqaS5gF/GRFzkkTxQ2AmuXUn7gFmRMSuoV6vVWsQhaZduYxq37Irz5rBRSdPr21AZjaqNaoGMRPYGBGbImIHsASYnz4gtb41wL7sWaVuPrAkIvoi4hlgY/J8be+Z6+aWveZEofzIbE/hYWblyDJBTAGeS21vScoGkXSJpKeBG4BLKzz3Qkk9knp6e9urGWXz4rlVNT2Bp/Aws/I0/C6miLg5It4FXAFcVeG5t0ZEd0R0T5rUnrd35msUlYyhyFtwxxqPozCzIWWZILYCU1PbhydlQ1kCfKzKc9veU9ecVVWNIr8OhROFmRXKMkGsBqZLmiZpLHAOsDR9gKR0j+lcIN84vhQ4R9I4SdOA6cCqDGNtGdXWKPp815OZFejM6okjol/SAmAF0AHcHhFrJV0N9ETEUmCBpNOAncArwAXJuWsl/RhYB/QDl5S6g8n29tQ1Zw08rmRkdn7A3WH7j+dnCz7E5AnjM4jOzEYDD5RrExd9r4cVa1+s+DwnCrPW5tlcbUA161CAE4VZq3KCsL1UMyEgOFGYtRonCBuSE4VZe3OCsGFVO4XH1APexk//8s+cKMxGKScIK1u1NQonCrPRyQnCKlZtohjXIe70NONmo4YThFXNfRRmrc0rylnV8pMCVur3r77FzGt/wTd+/mQGUZlZPbgGYRVxjcKstbiJyWqu2kThpVDNmosThGWm2kQBcNO5x/PR4/Za5sPM6sgJwjLnRGE2OjlBWN2MJFH81z8/lnP/tKt2wZjZsJwgrO5Gkij+6iNH8ZUz/qiG0ZjZUJwgrGFGkig8Otsse04Q1nDVzvUEHp1tliUnCGsa1a5HASDge1+YyUnvnlTboMzamBOENZ2RJApwh7ZZrThBWNOaee09vPR6X9XnO1GYjUzDEoSkOcCNQAdwW0QsLth/GfAFoB/oBT4XEb9L9u0CHk8OfTYi5pV6LSeI0c8d2mb115AEIakDeAo4HdgCrAY+HRHrUsfMBh6KiO2SvgicEhGfSva9ERH7lft6ThCtYySJAjzwzqwSjZrNdSawMSI2RcQOYAkwP31ARNwbEduTzQeBwzOMx0aJameQzVtwxxq6Fi7zTLJmI5RlgpgCPJfa3pKUDeXzwPLU9nhJPZIelPSxYidIujA5pqe3t3fkEVtTyScKqbrz/+6Xm+hauIxpC5excqN/P8wq1dnoAAAknQd0Ayenio+MiK2SjgJ+KenxiHg6fV5E3ArcCrkmproFbHX1zHW52kS1HdoBnHfbKsDNT2aVyLIGsRWYmto+PCkbRNJpwCJgXkQM/PVHxNbk+ybgPuCEDGO1UWDVotPYvHguZx57cNXP4eYns/Jl2UndSa6T+lRyiWE1cG5ErE0dcwLwE2BORGxIlR8AbI+IPkkHAb8B5qc7uAu5k7o9jbRD24PvrN2V6qTOrIkpIvolLQBWkLvN9faIWCvpaqAnIpYCfwvsB/yjcg3N+dtZ/wi4RdJucrWcxaWSg7WvfGd2tQPv0s1PV541g4tOnl7L8MxGNQ+Us5Yz0lqFV72zduKR1NaWRjJBYJ5rFdbqnCCsrY103idwX4W1LicIs8RIm5/A049ba3GCMCtQi+Yn8GSBNvo5QZgN4aLv9bBi7Ysjfh43Qdlo5QRhVoZa1Sq8praNJk4QZhWoVa0C3ARlzc8JwqxKtbgDCjy2wpqXE4RZDdTiDqjhuMZh9eYEYVZDtWyCKocH61mWnCDMMlKrju1quLZhteAEYVYH9WiCGo4H8VmlnCDM6qjahY2y4jEaVooThFmDNLIJajju2zBwgjBravXu9C7FTVTtxwnCbBRqhj4NcBNVq3OCMGsRzVTb8F1UrcEJwqzFubZh1XKCMGtDzdRB7g7x5tWwBCFpDnAj0AHcFhGLC/ZfBnwB6Ad6gc9FxO+SfRcAVyWHXhMR3y31Wk4QZuVpltoGOHE0g4YkCEkdwFPA6cAWYDXw6YhYlzpmNvBQRGyX9EXglIj4lKQDgR6gGwjgYeD9EfHKUK/nBGFWvWZKGgA3nXs8Hz1uSqPDaAulEkRnhq87E9gYEZuSIJYA84GBBBER96aOfxA4L3l8JnB3RGxLzr0bmAP8MMN4zdrW5sVz9yprZBPVgjvWsOCONXuVu8ZRX1kmiCnAc6ntLcCfljj+88DyEuf63wmzOnrmur2TRqPvorpu+VNct/ypvco9fiMbWSaIskk6j1xz0skVnnchcCHAEUcckUFkZpZ2y/lFWyIa3iHetys4+8aVe5X7rqqRyTJBbAWmprYPT8oGkXQasAg4OSL6UueeUnDufYXnRsStwK2Q64OoRdBmVrlmrG1ArgPzvNtWFd3ncRzDy7KTupNcJ/Wp5D7wVwPnRsTa1DEnAD8B5kTEhlT5geQ6pk9Mih4h10m9bajXcye12ejQDImjlHZbU7yRt7meDfw3cre53h4R10q6GuiJiKWS7gH+GHg+OeXZiJiXnPs54KtJ+bUR8felXssJwmx0a7ZZcAu1anOVB8qZ2ajV7DUOGN1rjjtBmFlLarbxG0Np5nEdThBm1lYafVdVJRrdWe4EYWbG6GiuSqvHwEAnCDOzYYyW5qq8WnWaO0GYmVVptDRXVdvP0ai5mMzMRr1igwDzmqnW8dc/erTmHeGuQZiZZWDGVcvZ0b+7Ia9dbPLFobgGYWZWZ09dc9aQ+7LqLBfwd+ceX7Pnc4IwM6uzoSY9zKu26aqzQzVtZnKCMDNrMqWaiEp1mvfvrm2XgROEmdkoUqrTvNbG1O2VzMxsVHGCMDOzopwgzMysKCcIMzMrygnCzMyKcoIwM7OiWmaqDUm9wO9G8BQHAX+oUTi15Lgq47gq47gq04pxHRkRRaeEbZkEMVKSeoaaj6SRHFdlHFdlHFdl2i0uNzGZmVlRThBmZlaUE8QetzY6gCE4rso4rso4rsq0VVzugzAzs6JcgzAzs6KcIMzMrKi2TxCS5khaL2mjpIV1fu2pku6VtE7SWklfSsr/RtJWSWuSr7NT51yZxLpe0pkZxrZZ0uPJ6/ckZQdKulvShuT7AUm5JH0riesxSSdmFNPRqWuyRtJrkr7cqOsl6XZJL0l6IlVW8TWSdEFy/AZJF2QU199K+pfkte+U9I6kvEvSm6lr9+3UOe9Pfgc2JrErg7gqfu9q/Tc7RFw/SsW0WdKapLye12uoz4f6/Y5FRNt+AR3A08BRwFjgUeCYOr7+ocCJyeMJwFPAMcDfAP+xyPHHJDGOA6YlsXdkFNtm4KCCshuAhcnjhcD1yeOzgeXkVjz8APBQnd67F4AjG3W9gA8DJwJPVHuNgAOBTcn3A5LHB2QQ1xlAZ/L4+lRcXenjCp5nVRKrktjPyiCuit67LP5mi8VVsP8bwNcacL2G+nyo2+9Yu9cgZgIbI2JTROwAlgDz6/XiEfF8RDySPH4deBIotV7gfGBJRPRFxDPARnI/Q73MB76bPP4u8LFU+T9EzoPAOyQdmnEspwJPR0Sp0fOZXq+I+BWwrchrVnKNzgTujohtEfEKcDcwp9ZxRcTPI6I/2XwQOLzUcySxTYyIByP3KfMPqZ+lZnGVMNR7V/O/2VJxJbWATwI/LPUcGV2voT4f6vY71u4JYgrwXGp7C6U/oDMjqQs4AXgoKVqQVBNvz1chqW+8Afxc0sOSLkzKDo6I55PHLwAHNyCuvHMY/Efb6OuVV+k1akSMnyP3n2beNEm/lXS/pFlJ2ZQklnrEVcl7V+/rNQt4MSI2pMrqfr0KPh/q9jvW7gmiKUjaD/gp8OWIeA34H8C7gOOB58lVcevtpIg4ETgLuETSh9M7k/+SGnKPtKSxwDzgH5OiZrhee2nkNRqKpEVAP/CDpOh54IiIOAG4DLhD0sQ6htSU713Kpxn8j0jdr1eRz4cBWf+OtXuC2ApMTW0fnpTVjaR9yL35P4iI/w0QES9GxK6I2A38T/Y0i9Qt3ojYmnx/CbgzieHFfNNR8v2leseVOAt4JCJeTGJs+PVKqfQa1S1GSZ8FPgp8JvlgIWnCeTl5/DC59v0ZSQzpZqhM4qrivavn9eoEPg78KBVvXa9Xsc8H6vg71u4JYjUwXdK05L/Sc4Cl9XrxpH3zfwFPRsQ3U+Xp9vs/B/J3VywFzpE0TtI0YDq5jrFax7WvpAn5x+Q6OJ9IXj9/B8QFwD+l4vqL5C6KDwCvpqrAWRj0X12jr1eBSq/RCuAMSQckzStnJGU1JWkOcDkwLyK2p8onSepIHh9F7hptSmJ7TdIHkt/Tv0j9LLWMq9L3rp5/s6cB/xIRA01H9bxeQ30+UM/fsZH0srfCF7me/6fI/SewqM6vfRK56uFjwJrk62zge8DjSflS4NDUOYuSWNczwrskSsR1FLm7Qx4F1uavC/BO4BfABuAe4MCkXMDNSVyPA90ZXrN9gZeB/VNlDble5JLU88BOcu26n6/mGpHrE9iYfP2HjOLaSK4dOv979u3k2E8k7/Ea4BHg36Wep5vcB/bTwE0kMy/UOK6K37ta/80Wiysp/w5wccGx9bxeQ30+1O13zFNtmJlZUe3exGRmZkNwgjAzs6KcIMzMrCgnCDMzK8oJwszMinKCMCtC0hvJ9y5J59b4ub9asP3/avn8ZrXiBGFWWhdQUYJIRuCWMihBRMSfVRiTWV04QZiVthiYpdzc/38tqUO5tRVWJxPMXQQg6RRJD0haCqxLyn6WTHa4Nj/hoaTFwNuS5/tBUpavrSh57ieUW1fgU6nnvk/ST5Rb0+EHyShbs0wN95+OWbtbSG69go8CJB/0r0bEn0gaB/xa0s+TY08E3hu56akBPhcR2yS9DVgt6acRsVDSgog4vshrfZzcpHXvAw5KzvlVsu8E4Fjg98CvgQ8BK2v/45rt4RqEWWXOIDffzRpyUy+/k9x8PACrUskB4FJJj5Jbf2Fq6rihnAT8MHKT170I3A/8Seq5t0RuUrs15Jq+zDLlGoRZZQT8VUQMmuxM0inAvxVsnwZ8MCK2S7oPGD+C1+1LPd6F/3atDlyDMCvtdXLLPeatAL6YTMOMpBnJjLeF9gdeSZLDe8gtAZm3M39+gQeATyX9HJPILYWZ9eyzZkPyfyFmpT0G7Eqair4D3EiueeeRpKO4l+JLS/4zcLGkJ8nNRvpgat+twGOSHomIz6TK7wQ+SG4W3QAuj4gXkgRjVneezdXMzIpyE5OZmRXlBGFmZkU5QZiZWVFOEGZmVpQThJmZFeUEYWZmRTlBmJlZUf8fZoF9q4AyPQwAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "CPU times: user 1h 1min 25s, sys: 2min 25s, total: 1h 3min 50s\n",
            "Wall time: 1h 3min 52s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:402: UserWarning: Couldn't retrieve source code for container of type Transliteration_EncoderDecoder. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.        , 0.52678156, 0.51426017, ..., 0.1783677 , 0.17832848,\n",
              "       0.17828813])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GM1Tj20omMi1",
        "colab_type": "text"
      },
      "source": [
        "### Training with Attention "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lxFLBqW1Ip4v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net_att = Transliteration_EncoderDecoder_Attention(len(eng_alpha2index), 256, len(hindi_alpha2index))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tdRpJUXNIwuv",
        "colab_type": "code",
        "outputId": "8e10ef5f-7e90-404b-b344-f3cc24cfdbb4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        }
      },
      "source": [
        "%%time\n",
        "loss_history = train_setup(model, lr=0.005, n_batches=2000, batch_size = 64, display_freq=100, device = device_gpu)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Iteration 1999 Loss 0.059731729328632355\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3xU9Z3/8dcnCUlACSgEFRCDClq0LWKktoLVWhVrV2prK97WrXYpbe1l/fWBuLjdrqVbcbu9WN0Kv+rPW720Utt0hdKKN6g3AiIKilxKFUQIF7kICYR8fn+cMziZzExmkjmZCfN+Ph555Mz33L5zMjmf+V6PuTsiIiKZKsl3BkREpHtR4BARkawocIiISFYUOEREJCsKHCIikpWyfGegK/Tv399ramrynQ0RkW5l0aJFm929OjG9KAJHTU0N9fX1+c6GiEi3YmZ/T5auqioREcmKAoeIiGRFgUNERLKiwCEiIllR4BARkawocLRj045GvjTjeTbtbMx3VkRECoICRztum7eShWu3ctsTK/OdFRGRghDpOA4zGwf8HCgFfuXutySsrwDuA04FtgCXuvvacN1HgBlAFdACnObujWZWDtwOnBWmT3X3WbnO+wk3zaGpueXA6wdefIsHXnyLirISVky7INenExHpNiIrcZhZKXAHcAEwArjMzEYkbHYtsM3djwd+CkwP9y0DHgAmuftJBEFiX7jPVGCTuw8Pj/tMFPmfP/lsLho58MDryh4ljB85kPk3nB3F6UREuo0oq6pGA6vcfY277wUeBsYnbDMeuDdcfhQ4x8wMOA9Y6u6vALj7FnffH253DfCjML3F3TdHkfkBVZX0rvigQNbU3ELvijIG9K6M4nQiIt1GlIFjEPB23Ot1YVrSbdy9GdgO9AOGA25mc81ssZlNBjCzvuF+PwjTf2tmRyQ7uZlNNLN6M6tvaGjo0BvYvKvpwPIVHzuGhrjXIiLFqlAbx8uAMcAV4e+LzeycMH0w8Jy7jwKeB36c7ADuPtPda929trq6zRxdGZlxVe2B5Tc37uQHnzu5Q8cRETmYRBk41gNHx70eHKYl3SZs1+hD0Ei+DnjW3Te7+25gNjAqXLcb+F24/2/D9MipZ5WISCDKwLEQGGZmQ8OeUBOAuoRt6oCrw+VLgCfd3YG5wIfNrFcYUD4JLA/X/ZGgsRzgHGB5VG/ghJvmHFh2D3pW1Ux5vFW6iEixiSxwhG0W1xEEgdeB37j7MjO72cwuCje7C+hnZquA64Ep4b7bgJ8QBJ8lwGJ3fzzc5wbg+2a2FLgK+D9RvYf5k1v3oFLPKhGRiMdxuPtsgmqm+LTvxS03Al9Mse8DBF1yE9P/DpyZ25wmN6CqdQ+qxn3qWSUiUqiN4wVpaP9D1LNKRIqeAkcaiW0Zf9v8PnOXbVQbh4gUNQWOFBKnHIkXtNGLiBQnBY4U5k8+m/NOaju28OjDerJgyqfykCMRkcKgwJHCgKpKqg+tSL5OjeMiUsQUONLYnNAQfvRhPRkxsCpPuRERKQyRdsft7mZcVUvNlMcPvH528tkEczCKiBQvBY4U4gNGzNAbgyEpa2+5sKuzIyJSMFRVlcLsb42hb68erdIG9a1k9rfH5ClHIiKFQYEjhRED+9CjpHW1VM8epYw4qk+eciQiUhgUONJoTBjHsX3PvhRbiogUDwWONP7twtZPup333bPykxERkQKixvEUko0c/8j3/0xFWQkrpl2Qp1yJiOSfShwpzJ98NqfVHNYq7Yzj+mlKdREpegocKQyoqmTD9sZWaUvXbdeocREpeqqqSiLVBIc7m5oPjO/QWA4RKVYqcSQxf/LZXDRyYNJ1g/v21FgOESlqChxJDKiqpHdF8sJYz3KN5RCR4hZp4DCzcWa2wsxWmdmUJOsrzOyRcP2LZlYTt+4jZva8mS0zs1fNrDJh3zozey2qvCdOcAgw+LCeGsshIkUvssBhZqXAHcAFwAjgMjMbkbDZtcA2dz8e+CkwPdy3jOB545Pc/STgLODAHdvMPg/siirvEExwmGjUkL68NPXTUZ5WRKTgRVniGA2scvc17r4XeBgYn7DNeODecPlR4BwLpp89D1jq7q8AuPsWd98PYGaHAtcD0yLMe1J1r2ygZsrjenSsiBS1KAPHIODtuNfrwrSk27h7M7Ad6AcMB9zM5prZYjObHLfPD4D/BnanO7mZTTSzejOrb2hoyDrzqYJDiaGxHCJS1Aq1cbwMGANcEf6+2MzOMbORwHHu/lh7B3D3me5e6+611dXVWWdg/uTkweHiUwZpLIeIFLUox3GsB46Oez04TEu2zbqwXaMPsIWgdPKsu28GMLPZwCiCdo1aM1sb5n2AmT3t7mflOvMDqtoGhyGH92RXU3OuTyUi0q1EWeJYCAwzs6FmVg5MAOoStqkDrg6XLwGedHcH5gIfNrNeYUD5JLDc3X/p7gPdvYagJPJmFEEjlYF9eyZtNBcRKSaRlTjcvdnMriMIAqXA3e6+zMxuBurdvQ64C7jfzFYBWwmCC+6+zcx+QhB8HJjt7m0fydfFplzwoXxnQUQk7yz4gn9wq62t9fr6+qz3S3x87IijqrjnmtPUxiEiRcHMFrl7m2qWQm0cL0jLN+xg+pw38p0NEZG80iSHWZq1eD2zFq/XczlEpGipxNFBxVDFJyKSjAJHB9T068WCKZ/KdzZERPJCgSOFdNOK7G9xNZCLSNFS4Egh1cjxL5w6iBEDq7o4NyIihUON4ynEjxwvKzGaW4I2jRvGnajShogUNZU4MvDPY489sKzuuCJS7FTiyMAvn1l9YFndcUWk2KnE0UHqjisixUqBIwP9Dy1v9VrdcUWkmClwZCB4KOEH1B1XRIqZAkcGGnY2tXr99rY9enysiBQtBY4MjDm+PyVhoaOyRwnjRw7U42NFpGgpcGSgZ49SwmEcNDW30LuiTFVVIlK0FDgysH3PXj50ZG8qy0q44mPH0LCrqf2dREQOUhrHkYHvnn8i897YyOrN7zPtcyfnOzsiInmlEkeGSs1oadHYDRGRSAOHmY0zsxVmtsrMpiRZX2Fmj4TrXzSzmrh1HzGz581smZm9amaVZtbLzB43szfC9FuizH+80hJjvwb9iYhEFzjMrBS4A7gAGAFcZmYjEja7Ftjm7scDPwWmh/uWAQ8Ak9z9JOAsYF+4z4/d/UTgFOAMM+uSeT9KSwx3VOoQkaIXZYljNLDK3de4+17gYWB8wjbjgXvD5UeBcywYbXcesNTdXwFw9y3uvt/dd7v7U2HaXmAxMDjC93BAaTgIUKUOESl2UQaOQcDbca/XhWlJt3H3ZmA70A8YDriZzTWzxWY2OfHgZtYX+AdgXrKTm9lEM6s3s/qGhoZOv5nd+/YD8O72xk4fS0SkOyvUxvEyYAxwRfj7YjM7J7YyrMp6CLjN3dckO4C7z3T3Wnevra6u7nSGXli9BYDbn1zV6WOJiHRnUXbHXQ8cHfd6cJiWbJt1YTDoA2whKJ086+6bAcxsNjCKD0oXM4GV7v6z6LL/gS/NeP7A8iP1b/NI/duaVl1EilaUJY6FwDAzG2pm5cAEoC5hmzrg6nD5EuBJD+Yrnwt8OOxFVQZ8ElgOYGbTCALMdyLMeytjju9PWTjnSGWZphwRkeIWWeAI2yyuIwgCrwO/cfdlZnazmV0UbnYX0M/MVgHXA1PCfbcBPyEIPkuAxe7+uJkNBqYS9NJabGZLzOwrUb2HmJ49Stkf9qZq2q8pR0SkuEU6ctzdZwOzE9K+F7fcCHwxxb4PEHTJjU9bB1iy7aP03p69nFZzOC+t3crnTxmkKUdEpKhpypEMfPe8E1iz+X1eWruV755/Akf16ZnvLImI5E2h9qoqOKVhG0fzfo3jEJHipsCRofcbmwFo2KlxHCJS3BQ4MvTE6xsBuOe5tfnNiIhInqmNIwOXznzhwHLdKxuoe+VxjeMQkaKlEkcGzjiuHz1KgzaOCo3jEJEip8CRgZ7lpQcaxffq0bEiUuQUODKwfc8+zj4hmO/q/JOP1DgOESlqauPIwHfPO4Gm5haeXNHAV8YMpbbm8HxnSUQkb1TiyFBZ2MbRrAc5iUiRU+DIUFlJcKk0AFBEip0CR4ZiI8dv/t9lbNIgQBEpYgocGYp1x31z4y6mz34jz7kREckfNY5n4Mq7XmRfXBXVrJfXM+vl9RoEKCJFSSWODKRq1VBrh4gUIwWODPzislM45vBerdJq+vVigUaPi0gRUuDIwGG9ytnvQfkibCNnf4t3i9Hjm3Y08qUZz6tBX0RyRm0cGTppYBXvbNvDF04dTEVZaUFPr75pRyMT71/Eu9v38O6OYJT76T+cx4hBVdz9T6d1i4AnkgsLVjZw5V0vZbRteanx++vOYMRRfSLOVfcXaeAws3HAz4FS4FfufkvC+grgPuBUYAtwqbuvDdd9BJgBVAEtwGnu3mhmpwL3AD0JHkv7bXePtLnBgBlX1fKhf/sTh/Uq518/86EoT5eV+H+M2y8fyeiafpz/02fZtmdfq+1agNfW72D0D+ex9pYL85BTkdzYtKORCTOfZ83m3Tk97t79zmd+vqBNugH3f2U0Y44Pph364yvr+eZDSwA4vvoQHpx4etF9GYsscJhZKXAHcC6wDlhoZnXuvjxus2uBbe5+vJlNAKYDl5pZGcHzxq9y91fMrB8QuxP+Evhn4EWCwDEOmBPV+4APGsHLSo19+1uiPFVasQ/s7ZeP5Nj+h3LRbQtojlt/3YNLMjpOzRRNCy/Z2bSjkYv/56+sfy91SbtvZRnvNTanXN+eb37qWC44eWCbz3W+OXDlr5KXWlY1vM/oH87r2gzFufGC4Xz1k8O6/LxRljhGA6vcfQ2AmT0MjAfiA8d44Pvh8qPA7WZmwHnAUnd/BcDdt4THOAqocvcXwtf3AZ8j4sARU1Zi7M/TlCPx33IyDRDpNDXnLwBK95FJwIjpTNAA+MWTa/jFk2s6dYxi86M5b/KjOW8eeH3ikb2579rRkZeAogwcg4C3416vAz6Waht3bzaz7UA/YDjgZjYXqAYedvdbw+3XJRxzULKTm9lEYCLAkCFDOv1mAMpKS1qN5+gqNVMe7/JzisR/WZHu4Y13dx4oAUXZZlOojeNlwBjgNGA3MM/MFgHbMz2Au88EZgLU1tbm5G7fo8TY39K139SjChqxkfAi8RQsDh6xNpsoqrOiDBzrgaPjXg8O05Jtsy5s1+hD0Ei+DnjW3TcDmNlsYBRBu8fgdo6Zc+/t3gtAaakdNJMc7tvv1Ex5XA3lAsAvn17J9D+92f6G3dz5Jx3BjKtqW6UNvfFxou1ek1+x6qxc/q9HGTgWAsPMbCjBzX0CcHnCNnXA1cDzwCXAk+4eq6KabGa9gL3AJ4GfuvsGM9thZqcTNI7/I/CLCN8DALMWrWPcyUfRo6SEfV3YxnHCTZ1vujEj7T/FglUNB3qLSHHJpv2i0B1RVcGOsCdhVWUPystKGDGwqk2QSOZvP0p+Qz3YAkouvyhGFjjCNovrgLkE3XHvdvdlZnYzUO/udcBdwP1mtgrYShBccPdtZvYTguDjwGx3j9XZfJ0PuuPOIYKG8RNumtOq8fgvr286UGW0bfdeNu1s7JLudy1ZBqnY4MQWD5bPHRF8uzr2xsdJdahYbxGVPKIVf5POV0+YeMvf2c6Fty3o8LQ58Z+X2P9GWQnE/m3Ky0p4M6HX3lfvr+cvyzfSq7yUYQN68+r69ygpKWFvmo4apSVQXlpC7TGHs/itbRxSUcZLUz/dwVxnJ1VAydZX76+nuncll48ewoMvvcWfl73LeScdeeB1w87GjAJc/PHmLtuYdT5uvGB41vukYhEPgSgItbW1Xl9fn/H2m3Y0Mm3269QteQeAirISxp18JE+9sYkdjc1c+bEhTLv4w1FlFwj+sT9zW9s+5ckkK37Hy7SdpFCDx6YdjVxy53O8tXVPq/RDy0vYtbelIG7EyaQbfDawTyW/v+6MvPT/T/xilI1C/YwUs+E3zUkbfGM68rczs0Xu3ubmosCRwtTHXuXXL76VdpuoxkIMnzqHvSnGi5QajB1ezT1fHp3x8TINQrO/PaZVD4xNOxq57qGXuf3yU/I6wOnaexYy741Nabep7FHC777+iYIY9bv8ne0dHotw++Uj+exHknYUzImOdrYY0Luiy77pS+ekqmJT4MhSRwLHV++vZ8W7O9nb3MLpx/XjudWbeXd7MH1HqcFnPzqQqRd+KOc31KhKByfeNJvG5sz+1rdfPpK3t+4+0FhaavDHb43p8ptyugCaShmkvWEPPqwn1b0rmHHVqTn922UztUVnJI5iztTyd7Zz8R0LaNqf+T4KFtKpwGFmhwB73L3FzIYDJwJz3H1fO7sWhI4EDoBvPfQyS9e9x4btjUmL9rkucWRahfBAB24co3/4BJt2NnU0awec3IXzXXX1+JX4Kq/l72zn0hkv8Mik09MGzHw2MF/ziSE8Ur+evr16sP69xqSlleXvbOdzty9gbxbxV9VREtPZwLEIGAscBvyVoNF6r7tfkeuMRqEzgePV9dt5ZOLpXHTHggMlDgOO7FPJH3JcR53JjbJ3RRmv/sf5HTr+6B8+Qd9ePfhbwy72dWI4Su+KUuZ996zIgsewqbPzMtAyGQMej6vCO5i7rSpgSKJUgSPTXlXm7rvN7Frgf9z9VjM76EcJxW5dY299qlVJwIEN2xsZO/2pnJY4epTQ7g29qRNzZcWqHTraKyNmZ9N+Rv9wXiRtPP/1p9cLJmhA8LdONvHdwUZBQ7KR6fM4zMw+DlwBxL4Wl0aTpcJiwPzJZ3Nkn9bfrnuVlzI/xw9ySne7PPqwnpx/0hFtujh2xIyrajn/pCM6fZym5pacjDWJqZnyOHc8fXDOVVReVlKwN+dCzZcUrkxLHN8BbgQeC8diHAs8FV22CkOsGm9AVSWbdrSuw969N7ffutNVU5WXGvNv+FSnzxFvxlW1nS55QG4mS8ym+ie+wTZX7TZRi78xJ7tJ5/N9KGhIR2QUONz9GeAZADMrATa7+7eizFjBCAfVnTmsP0vefo/39nzQZ2f8yKBnVWe117axYEpug0ZMLHgsf2cH69/bk3KQYHs6MyL1j6+szzhoJJ4jvsdPqms4oHcFQJfemOMHX2YisefSV++v59k3GxjQu5Ihh/di8VvbeH9vFt2hMqAeU9IZGQUOM3sQmATsJ2gYrzKzn7v7f0WZuXyLv48+v2Zrm2/Xf1jyDn967d1OlTjaCxrVh5ZH2oOpvZtbpqWS2DM+HvtG+2Mplr+znfG/WMC+LAJVLACkkm3gio3mXdOwi+dWb8lq33TaG4yZifb270xvs1zkTyTTqqoR7r7DzK4gmOJjCrAIOKgDBwQFjnTdZDvbztFeg/ioYw7r1PE7K/4m094Nq6m5ha/et6jdarVr7lmYddDI9bfj9m6emcxTdERVBbXHHM7Sde9lPC9SLqh6SfIt08DRw8x6EDw06XZ332dmhdP1JWLzJ5/NtNmv88cl77QqhXxh1KBOlQZOuGlO2qBRXlZSUN8OB/SuaLfK5+1tew4EmLW3XMimHY18+Z6FrNy4g2xrW/J5g4zNUzT8pjn07FHC1z55HL98ZjV79rXkpIOCSHeWaeCYAawFXgGeNbNjgB1RZapghFFiQFUlvSvKWgWNHqXGll2dqzdv7xttod2gXpr6aS6b+TzPr9ma0fZn3DKvwwPjCuVbdfzfYNJZx+cxJyKFI6PuuO5+m7sPcvfPeODvQG77ohao4Em2sHlXE2OH9TuQvm+/0+/Q9PXu7Uk3ncZLU8/p1LGj8tDEj2e8bUeCxvknHVEwQUNEksu0cbwP8O/AmWHSM8DNZPFEvu7I48oYT69oaNPOMWvxemYtXt+hLrnttRfkc1LB9mRSZdXR4xZS1ZyIJJfpAMC7gZ3Al8KfHcD/iypThST2gNV0U7N0ZKLI2d8aw5FVyYPDWScU9oOVXpr6acrLMv3otG/tLRey9pYL1T1UpJvI9L//OHf/d3dfE/78B3BslBkrBJnGg46Ms7j4f57j3R3Jq3KymTI9X96cdgHlZSX06VnGoeUdn0QgFyPYRaRrZdo4vsfMxrj7AgAzOwPY084+BwWzoPfT3hTzJw3sU9mhaqXHvv6JNs/IKDE4c3hhlzbixTccd2QUutoyRLqnTAPHJOC+sK0DYBvBs8IParESR6w7buyJgPHe2d7ICTfNSdvGkfhApE07GpM+WKnFg7aU7ijWNpFq/INGKoscPDKdcuQV4KNmVhW+3mFm3wGWptvPzMYBPyeYEPFX7n5LwvoK4D7gVGALcKm7rzWzGuB1YEW46QvuPinc5zLgXwk6y74DXOnumzN5Hx1h2IHuuKm018Zx27yVLFy7ldueWMlvF61LO79Tj1JLua47yNVzmkWkcGVa4gCCgBH38nrgZ6m2NbNS4A7gXGAdsNDM6tx9edxm1wLb3P14M5sATAcuDdetdveRCccsIwhEI9x9s5ndClwHfD+b99ERm3c1cdbw/jz9ZusYVdOvF7+ZlLyLauKI8wfaeRQtwF8jmpdKRCRXOtM1pr2vxqOBVWFj+l7gYWB8wjbjgXvD5UeBcyw2cCL1OQ04JNyuiqDUEYn47rgzrqrlr0nmNFq7ZTdjpyefKHj+5LO5aOTAjM9XXmoF3Q1XRASyLHEkaK/P0SDg7bjX64CPpdrG3ZvNbDsQG2U31MxeJuj6e5O7zw+nOvka8CrwPrAS+Eayk5vZRGAiwJAhQzJ+U22PE/w+4aY5KR8wlGq+qtH/OS+rc5194oCsthcRyYe0JQ4z22lmO5L87AQy/yqdvQ3AEHc/haBK7EEzqwrny/oacEp4/qUEzwlpw91nunutu9dWV3esp1J808X8yWdz3oi2XUcv/PCROSklFNq8VCIiqaQtcbh7704cez1wdNzrwWFasm3Whe0XfYAtHrQ2N4V5WGRmq4HhhNVj7r4awMx+QzBTb+QGVFVSnWRq76Xrkg+ez+bJeOVlJQU3L5WISCq5G/7b1kJgmJkNNbNyYAJQl7BNHR90670EeNLd3cyqw8Z1wqcNDgPWEASaEWYWK0KcS9D7KhKJFVMPvdS2cTs2G2xioMj0yXgKGiLS3UQWONy9maDH01yCm/tvwsfO3mxmF4Wb3QX0M7NVBFVSsdLDmcBSM1tC0Gg+yd23uvs7wH8QzNC7FBgJ/GdU7wE+mOQQ4IUbz+G8JCOdx510RIeey9H/0HIFDRHpdjrTON4ud58NzE5I+17cciPwxST7zQJmpTjmncCduc1pco1797N28/ts2tnIgN6VjL31qaQliT8t28hTKxqymuiwxKD+pnNzmV0RkS4RZVVVt7dm8/vs2bef255YCQQN5MnaOQb0ruCxr3+CL814nk07G9O2b3x0cB+uPP0Yzk3S0C4i0h1YR2Z27W5qa2u9vr4+4+1TPSq2oqyEfftbaElzyb4wahCzFif2AfiA5mcSke7CzBa5e5vunipxJBEbuFcSNm9U9ihh/MiBzL/hbM4c1p9ePVLPBpsuaIiIHAwUOJKIzU3V4kH/36bmFnpXlDGgdyX3XPMxLh41qEPHLe/m81CJiEDEjePd2eZdTQzqW0llj1I+flx/GnY2tlpXUWo0pRhJnkyJdey5HSIihUZtHGlce89CNu5s5H+/OfZAWqr2j0x05BGzIiL5ojaODkgWUjsTaDsy1kNEpNAocLTDEiYBXnBDx6ubNPOtiBwMFDjSSFa6GHtr8inURUSKhQJHOxKfDjJ/cseqm2Z/e0wOciMikn8KHGkka80YUFXJ50/JvjvuiKP6tL+RiEg3oO647Ug28uJ3L2c+yK+sBPr2Ks9dhkRE8kwljog1t2gyQxE5uChwpJGq5+3sb2XXXpHNQ51ERAqdAkd7ElvHgREDM2+v6OizOkRECpUCRxrphvr1OySzdos/LdvI2OnqwisiBw8FjnakmpZw0b+dy+DDegLQI83khX17lqnEISIHFQWONNqbXuSkgVVcefox/OEbY7jy9GOSBpnPfmSgRoyLyEEl0sBhZuPMbIWZrTKzKUnWV5jZI+H6F82sJkyvMbM9ZrYk/Lkzbp9yM5tpZm+a2Rtm9oVo30PqdTOuqmXa505mxMAqpn3uZLCghHLCEYdSFl7Zhl1NUWZPRKTLRTaOw8xKgTuAc4F1wEIzq3P35XGbXQtsc/fjzWwCMB24NFy32t1HJjn0VGCTuw83sxLg8KjeQ7b+9iM93U9EDn5RljhGA6vcfY277wUeBsYnbDMeuDdcfhQ4xyzdd3wArgF+BODuLe6+OYd5bkOPXhIRaS3KwDEIeDvu9bowLek27t4MbAf6heuGmtnLZvaMmY0FMLO+4bofmNliM/utmR2R7ORmNtHM6s2svqGhIUdvSURECrVxfAMwxN1PAa4HHjSzKoKqtcHAc+4+Cnge+HGyA7j7THevdffa6urqDmWiCJ5xJSKStSgDx3rg6LjXg8O0pNuYWRnQB9ji7k3uvgXA3RcBq4HhwBZgN/C7cP/fAqOiegNhvqI8vIhItxNl4FgIDDOzoWZWDkwA6hK2qQOuDpcvAZ50dzez6rBxHTM7FhgGrPGgf+wfgbPCfc4BlhMRTzsEUESkOEXWq8rdm83sOmAuUArc7e7LzOxmoN7d64C7gPvNbBWwlSC4AJwJ3Gxm+4AWYJK7bw3X3RDu8zOgAfhyVO8B1DguIpIo0mnV3X02MDsh7Xtxy43AF5PsNwuYleKYfycILJFTG4eISFuF2jheMNTEISLSmgJHGipxiIi0pcDRDlMrh4hIKwocIiKSFQWONNQdV0SkLQWO9qimSkSkFQWONNQ4LiLSlgJHO1TgEBFpTYEjDRU4RETaUuBohwYAioi0psCRjoocIiJtKHC0QwMARURaU+AQEZGsKHCkoQGAIiJtKXC0Q43jIiKtKXCkoQGAIiJtKXC0QyUOEZHWFDjSUIFDRKStSAOHmY0zsxVmtsrMpiRZX2Fmj4TrXzSzmjC9xsz2mNmS8OfOJPvWmdlrUeYf1B1XRCRRZM8cN7NS4A7gXGAdsNDM6tx9edxm1wLb3P14M5sATAcuDdetdveRKY79eWBXVHmPcTVyiIi0EWWJYzSwyt3XuPte4JNuz9UAAAvkSURBVGFgfMI244F7w+VHgXPM0rcqmNmhwPXAtBznN8X5uuIsIiLdR5SBYxDwdtzrdWFa0m3cvRnYDvQL1w01s5fN7BkzGxu3zw+A/wZ2R5JrERFJK7Kqqk7aAAxx9y1mdirwezM7CTgWOM7d/yXWHpKKmU0EJgIMGTKkQ5lQRZWISFtRljjWA0fHvR4cpiXdxszKgD7AFndvcvctAO6+CFgNDAc+DtSa2VpgATDczJ5OdnJ3n+nute5eW11dnbM3JSJS7KIMHAuBYWY21MzKgQlAXcI2dcDV4fIlwJPu7mZWHTauY2bHAsOANe7+S3cf6O41wBjgTXc/K6o3oLZxEZG2IquqcvdmM7sOmAuUAne7+zIzuxmod/c64C7gfjNbBWwlCC4AZwI3m9k+oAWY5O5bo8prOu201YuIFJ1I2zjcfTYwOyHte3HLjcAXk+w3C5jVzrHXAifnJKOpzhHlwUVEuimNHG+HyhsiIq0pcKSjRg4RkTYUONqhJg4RkdYUOEREJCsKHGmookpEpC0FjnaopkpEpDUFjjTUNi4i0pYCRzs0AFBEpDUFjjRcrRwiIm0ocLRD5Q0RkdYUONJQG4eISFsKHO1QE4eISGsKHGmoxCEi0pYCR7tU5BARiafAISIiWVHgSEM1VSIibSlwtEON4yIirSlwpOFqHRcRaSPSwGFm48xshZmtMrMpSdZXmNkj4foXzawmTK8xsz1mtiT8uTNM72Vmj5vZG2a2zMxuiTL/oKZxEZFEkQUOMysF7gAuAEYAl5nZiITNrgW2ufvxwE+B6XHrVrv7yPBnUlz6j939ROAU4AwzuyCq9yAiIm1FWeIYDaxy9zXuvhd4GBifsM144N5w+VHgHEszq6C773b3p8LlvcBiYHDOcx5HbRwiIq1FGTgGAW/HvV4XpiXdxt2bge1Av3DdUDN72cyeMbOxiQc3s77APwDzcp3xGDVxiIi0VZbvDKSwARji7lvM7FTg92Z2krvvADCzMuAh4DZ3X5PsAGY2EZgIMGTIkA5nxNTKISLSSpQljvXA0XGvB4dpSbcJg0EfYIu7N7n7FgB3XwSsBobH7TcTWOnuP0t1cnef6e617l5bXV3d6TcjIiKBKAPHQmCYmQ01s3JgAlCXsE0dcHW4fAnwpLu7mVWHjeuY2bHAMGBN+HoaQYD5ToR5B/Q8DhGRZCILHGGbxXXAXOB14DfuvszMbjazi8LN7gL6mdkq4Hog1mX3TGCpmS0haDSf5O5bzWwwMJWgl9bisKvuV6J6D837W3jxb1vYtLMxqlOIiHQ7kbZxuPtsYHZC2vfilhuBLybZbxYwK0n6OrpwaMXmXXvZ0djMbU+sZNrFH+6q04qIFLRCbRzPqxNumkNTc8uB1w+8+BYPvPgWFWUlrJimYSMiUtw05UgS8yefzUUjB1JaEhRuKnuUMH7kQObfcHaecyYikn8KHEkMqKqkd0UZLe5UlJXQ1NxC74oyBvSuzHfWRETyTlVVKWze1cQVHzuGy0cP4cGX3qJBDeQiIgBYMcwAW1tb6/X19fnOhohIt2Jmi9y9NjFdVVUiIpIVBQ4REcmKAoeIiGRFgUNERLKiwCEiIllR4BARkawURXdcM2sA/t7B3fsDm3OYnVxRvrKjfGWnUPMFhZu3gzFfx7h7m+dSFEXg6Awzq0/WjznflK/sKF/ZKdR8QeHmrZjypaoqERHJigKHiIhkRYGjfTPznYEUlK/sKF/ZKdR8QeHmrWjypTYOERHJikocIiKSFQUOERHJigJHCmY2zsxWmNkqM5vSxec+2syeMrPlZrbMzL4dpn/fzNab2ZLw5zNx+9wY5nWFmZ0fYd7Wmtmr4fnrw7TDzewvZrYy/H1YmG5mdluYr6VmNirCfJ0Qd12WmNkOM/tOPq6Zmd1tZpvM7LW4tKyvkZldHW6/0syujihf/2Vmb4TnfszM+obpNWa2J+663Rm3z6nhZ2BVmHeLIF9Z/91y/T+bIl+PxOVprZktCdO78nqluj903WfM3fWT8AOUAquBY4Fy4BVgRBee/yhgVLjcG3gTGAF8H/huku1HhHmsAIaGeS+NKG9rgf4JabcCU8LlKcD0cPkzwBzAgNOBF7vw7/cucEw+rhlwJjAKeK2j1wg4HFgT/j4sXD4sgnydB5SFy9Pj8lUTv13CcV4K82ph3i+IIF9Z/d2i+J9Nlq+E9f8NfC8P1yvV/aHLPmMqcSQ3Gljl7mvcfS/wMDC+q07u7hvcfXG4vBN4HRiUZpfxwMPu3uTufwNWEbyHrjIeuDdcvhf4XFz6fR54AehrZkd1QX7OAVa7e7rZAiK7Zu7+LLA1yfmyuUbnA39x963uvg34CzAu1/ly9z+7e3P48gVgcLpjhHmrcvcXPLj73Bf3XnKWrzRS/d1y/j+bLl9hqeFLwEPpjhHR9Up1f+iyz5gCR3KDgLfjXq8j/Y07MmZWA5wCvBgmXRcWN++OFUXp2vw68GczW2RmE8O0I9x9Q7j8LnBEHvIVbwKt/6Hzfc0g+2uUj2t3DcE305ihZvaymT1jZmPDtEFhXroiX9n83br6eo0FNrr7yri0Lr9eCfeHLvuMKXAUMDM7FJgFfMfddwC/BI4DRgIbCIrKXW2Mu48CLgC+YWZnxq8Mv1XlrY+3mZUDFwG/DZMK4Zq1ku9rlIyZTQWagV+HSRuAIe5+CnA98KCZVXVhlgru75bgMlp/Oeny65Xk/nBA1J8xBY7k1gNHx70eHKZ1GTPrQfCh+LW7/w7A3Te6+353bwH+Lx9UrXRZft19ffh7E/BYmIeNsSqo8Pemrs5XnAuAxe6+Mcxn3q9ZKNtr1GX5M7N/Aj4LXBHecAirgraEy4sI2g+Gh3mIr86KJF8d+Lt15fUqAz4PPBKX3y69XsnuD3ThZ0yBI7mFwDAzGxp+g50A1HXVycP607uA1939J3Hp8e0DFwOx3h51wAQzqzCzocAwgga5XOfrEDPrHVsmaFh9LTx/rEfG1cAf4vL1j2GvjtOB7XFF6ai0+iaY72sWJ9trNBc4z8wOC6tpzgvTcsrMxgGTgYvcfXdcerWZlYbLxxJcnzVh3naY2enh5/Qf495LLvOV7d+tK/9nPw284e4HqqC68nqluj/QlZ+xzrTuH8w/BD0R3iT45jC1i889hqCYuRRYEv58BrgfeDVMrwOOittnapjXFXSy10aafB1L0FvlFWBZ7LoA/YB5wErgCeDwMN2AO8J8vQrURnzdDgG2AH3i0rr8mhEErg3APoJ642s7co0I2hxWhT9fjihfqwjquWOfszvDbb8Q/o2XAIuBf4g7Ti3BjXw1cDvhDBQ5zlfWf7dc/88my1eYfg8wKWHbrrxeqe4PXfYZ05QjIiKSFVVViYhIVhQ4REQkKwocIiKSFQUOERHJigKHiIhkRYFDJAtmtiv8XWNml+f42P+a8Pq5XB5fJFcUOEQ6pgbIKnCEI47TaRU43P0TWeZJpEsocIh0zC3AWAuevfAvZlZqwbMtFoYT830VwMzOMrP5ZlYHLA/Tfh9OErksNlGkmd0C9AyP9+swLVa6sfDYr1nwXIdL4479tJk9asEzNX4djioWiVR734BEJLkpBM+L+CxAGAC2u/tpZlYB/NXM/hxuOwo42YNpwAGucfetZtYTWGhms9x9ipld5+4jk5zr8wST/X0U6B/u82y47hTgJOAd4K/AGcCC3L9dkQ+oxCGSG+cRzAe0hGCK634E8xUBvBQXNAC+ZWavEDz/4ui47VIZAzzkwaR/G4FngNPijr3Og8kAlxBUoYlESiUOkdww4Jvu3mqSODM7C3g/4fWngY+7+24zexqo7MR5m+KW96P/aekCKnGIdMxOgsd2xswFvhZOd42ZDQ9nEE7UB9gWBo0TCR7lGbMvtn+C+cClYTtKNcEjTaOcyVckLX07EemYpcD+sMrpHuDnBNVEi8MG6gaSPyL0T8AkM3udYHbXF+LWzQSWmtlid78iLv0x4OMEsxI7MNnd3w0Dj0iX0+y4IiKSFVVViYhIVhQ4REQkKwocIiKSFQUOERHJigKHiIhkRYFDRESyosAhIiJZ+f+GBrfnQLmmXgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "CPU times: user 1h 17min 42s, sys: 1min 44s, total: 1h 19min 26s\n",
            "Wall time: 1h 19min 24s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:402: UserWarning: Couldn't retrieve source code for container of type Transliteration_EncoderDecoder_Attention. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "05F1-FwX6YVZ",
        "colab_type": "text"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v3TWC7zhAn3z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def test(net, word, device = 'cpu'):\n",
        "    net = net.eval().to(device)\n",
        "    outputs = infer(net, word, 30, device)\n",
        "    hindi_output = ''\n",
        "    for out in outputs:\n",
        "        val, indices = out.topk(1)\n",
        "        index = indices.tolist()[0][0]\n",
        "        if index == 0:\n",
        "            break\n",
        "        hindi_char = hindi_alphabets[index+1]\n",
        "        hindi_output += hindi_char\n",
        "    print(word + ' - ' + hindi_output)\n",
        "    return hindi_output"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bT8bibYl7CgX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calc_accuracy(net, device = 'cpu'):\n",
        "    net = net.eval().to(device)\n",
        "    predictions = []\n",
        "    accuracy = 0\n",
        "    for i in range(len(test_data)):\n",
        "        eng, hindi = test_data[i]\n",
        "        gt = gt_rep(hindi, hindi_alpha2index, device)\n",
        "        outputs = infer(net, eng, gt.shape[0], device)\n",
        "        correct = 0\n",
        "        for index, out in enumerate(outputs):\n",
        "            val, indices = out.topk(1)\n",
        "            hindi_pos = indices.tolist()[0]\n",
        "            if hindi_pos[0] == gt[index][0]:\n",
        "                correct += 1\n",
        "        \n",
        "        accuracy += correct/gt.shape[0]\n",
        "    accuracy /= len(test_data)\n",
        "    return accuracy"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dy1bQiORAs5o",
        "colab_type": "code",
        "outputId": "ef1a1609-c433-4427-d883-b5ca6bb2ba97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "accuracy = calc_accuracy(net) * 100\n",
        "accuracy_attn = calc_accuracy(model) * 100\n",
        "print('Accuracy w/o attention ', accuracy)\n",
        "print('Acurracy with attention', accuracy_attn)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy w/o attention  0.4905591630591632\n",
            "Acurracy with attention 73.15899267399266\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9FSgnNJBHm7l",
        "colab_type": "code",
        "outputId": "e866daf6-6071-41ed-f0ea-80ae9883506f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "model = torch.load('model_73.pt')\n",
        "model.eval()\n",
        "model"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Transliteration_EncoderDecoder_Attention(\n",
              "  (encoder_rnn_cell): GRU(27, 256)\n",
              "  (decoder_rnn_cell): GRU(512, 256)\n",
              "  (h2o): Linear(in_features=256, out_features=129, bias=True)\n",
              "  (softmax): LogSoftmax()\n",
              "  (U): Linear(in_features=256, out_features=256, bias=True)\n",
              "  (W): Linear(in_features=256, out_features=256, bias=True)\n",
              "  (attn): Linear(in_features=256, out_features=1, bias=True)\n",
              "  (out2hidden): Linear(in_features=129, out_features=256, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4iApzWytOIxo",
        "colab_type": "code",
        "outputId": "2bb2577d-0818-41f7-9b7e-178dd1bddd84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "hindi_alphabets[5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'अ'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "thM8btG8L5Hs",
        "colab_type": "code",
        "outputId": "93ba6de3-a484-45be-b09b-d63c2834e75d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "hindi_word = \"\"\n",
        "out = infer(model, 'OK', 30)\n",
        "for i in out:\n",
        "  val = torch.argmax(i)\n",
        "  index = val.item()\n",
        "  hindi_word = hindi_word + hindi_alphabets[index]\n",
        "print(hindi_word)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "औखऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀऀ\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9t9_GzzBQFup",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}