PersianG2P.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Loading data "
      ],
      "metadata": {
        "id": "n2UC5X0evaSC"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Ql5c0Cgu3qQ",
        "outputId": "00f2d5f7-1d57-4c7e-99ba-35f2b4500d72"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd drive/MyDrive"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yj6NUTpKv-Qe",
        "outputId": "ad0f58b8-aac8-4b56-f77a-6261e3ca568e"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv(\"data.csv\",encoding=\"utf-8\")"
      ],
      "metadata": {
        "id": "1Rv6-LODvZ2Q"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(data))\n",
        "data.head(30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "eykXoVKtv7HK",
        "outputId": "2bb86d7d-ec99-4ca6-a9ba-ea50b43542ef"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "75182\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           grapheme             phoneme\n",
              "0                ره   rahmatollAh'alayh\n",
              "1                ره         rahmatollAh\n",
              "2                 ص  salavAtollAh'alayh\n",
              "3                 ع       'alayhessalAm\n",
              "4                عج   'ajalallAhota'AlA\n",
              "5              ق.ا.        qanune'asAsi\n",
              "6              ق.م.        qanunemadani\n",
              "7              ق.م.         qablazmilAd\n",
              "8               ق.م         qablazmilAd\n",
              "9               ق‌م         qablazmilAd\n",
              "10                م              milAdi\n",
              "11             ه.ش.        hejriyeSamsi\n",
              "12              ه.ش        hejriyeSamsi\n",
              "13              ه‌ش        hejriyeSamsi\n",
              "14             ه.ق.       hejriyeqamari\n",
              "15              ه.ق       hejriyeqamari\n",
              "16              ه‌ق       hejriyeqamari\n",
              "17                ا               'alef\n",
              "18                ا                  'e\n",
              "19                ا                  'A\n",
              "20               اب                 'ab\n",
              "21              ابا                'ebA\n",
              "22              ابا                'abA\n",
              "23             اباء                ebA'\n",
              "24  اباالفضل‌العباس     abalfazle'abbAs\n",
              "25           ابابکر            'abAbakr\n",
              "26           ابابیل             'abAbil\n",
              "27          اباجعفر          'abAja'far\n",
              "28           اباحتی            'ebAhati\n",
              "29            اباحه              'ebAhe"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bb0ee707-092c-43f9-9ccf-884a41c7228c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>grapheme</th>\n",
              "      <th>phoneme</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ره</td>\n",
              "      <td>rahmatollAh'alayh</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ره</td>\n",
              "      <td>rahmatollAh</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ص</td>\n",
              "      <td>salavAtollAh'alayh</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ع</td>\n",
              "      <td>'alayhessalAm</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>عج</td>\n",
              "      <td>'ajalallAhota'AlA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ق.ا.</td>\n",
              "      <td>qanune'asAsi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>ق.م.</td>\n",
              "      <td>qanunemadani</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>ق.م.</td>\n",
              "      <td>qablazmilAd</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>ق.م</td>\n",
              "      <td>qablazmilAd</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>ق‌م</td>\n",
              "      <td>qablazmilAd</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>م</td>\n",
              "      <td>milAdi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>ه.ش.</td>\n",
              "      <td>hejriyeSamsi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>ه.ش</td>\n",
              "      <td>hejriyeSamsi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>ه‌ش</td>\n",
              "      <td>hejriyeSamsi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>ه.ق.</td>\n",
              "      <td>hejriyeqamari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>ه.ق</td>\n",
              "      <td>hejriyeqamari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>ه‌ق</td>\n",
              "      <td>hejriyeqamari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ا</td>\n",
              "      <td>'alef</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>ا</td>\n",
              "      <td>'e</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>ا</td>\n",
              "      <td>'A</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>اب</td>\n",
              "      <td>'ab</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>ابا</td>\n",
              "      <td>'ebA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>ابا</td>\n",
              "      <td>'abA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>اباء</td>\n",
              "      <td>ebA'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>اباالفضل‌العباس</td>\n",
              "      <td>abalfazle'abbAs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>ابابکر</td>\n",
              "      <td>'abAbakr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>ابابیل</td>\n",
              "      <td>'abAbil</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>اباجعفر</td>\n",
              "      <td>'abAja'far</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>اباحتی</td>\n",
              "      <td>'ebAhati</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>اباحه</td>\n",
              "      <td>'ebAhe</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bb0ee707-092c-43f9-9ccf-884a41c7228c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bb0ee707-092c-43f9-9ccf-884a41c7228c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bb0ee707-092c-43f9-9ccf-884a41c7228c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Deleting 0 to 19 rows:\n",
        "because they are abbreviations in persian"
      ],
      "metadata": {
        "id": "tEQVChfgwbkN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data=data.drop(data.index[:20])\n",
        "print(len(data))\n",
        "data.head(30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "iRWZLiHcwUSF",
        "outputId": "56f945cd-09ea-4a86-c5a5-1132e062bdbe"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "75162\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           grapheme           phoneme\n",
              "20               اب               'ab\n",
              "21              ابا              'ebA\n",
              "22              ابا              'abA\n",
              "23             اباء              ebA'\n",
              "24  اباالفضل‌العباس   abalfazle'abbAs\n",
              "25           ابابکر          'abAbakr\n",
              "26           ابابیل           'abAbil\n",
              "27          اباجعفر        'abAja'far\n",
              "28           اباحتی          'ebAhati\n",
              "29            اباحه            'ebAhe\n",
              "30        اباحه‌گری        'ebAhegari\n",
              "31            اباحی            'ebAhi\n",
              "32          اباحیگر         'ebAhigar\n",
              "33         اباحیگری        'ebAhigari\n",
              "34            ابادت           'ebAdat\n",
              "35            اباذر           'abAzar\n",
              "36             ابار             'abAr\n",
              "37           اباریق           'abAriq\n",
              "38           اباطیل           'abAtil\n",
              "39     اباعبدالرحمن  'abA'abdorrahmAn\n",
              "40       اباعبدالله     'abA'abdellAh\n",
              "41      اباعبدالملک   'abA'abdolmalek\n",
              "42            اباعد           'abA'ed\n",
              "43            اباقا            'AbAqA\n",
              "44         اباقاخان         'AbAqAxAn\n",
              "45           ابالسه          'abAlese\n",
              "46           ابالیس           'abAlis\n",
              "47             ابام             'abAm\n",
              "48             ابان             'abAn\n",
              "49            ابانت           'ebAnat"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-78aaa0c1-65eb-4d0e-8260-b90dc38530cb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>grapheme</th>\n",
              "      <th>phoneme</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>اب</td>\n",
              "      <td>'ab</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>ابا</td>\n",
              "      <td>'ebA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>ابا</td>\n",
              "      <td>'abA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>اباء</td>\n",
              "      <td>ebA'</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>اباالفضل‌العباس</td>\n",
              "      <td>abalfazle'abbAs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>ابابکر</td>\n",
              "      <td>'abAbakr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>ابابیل</td>\n",
              "      <td>'abAbil</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>اباجعفر</td>\n",
              "      <td>'abAja'far</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>اباحتی</td>\n",
              "      <td>'ebAhati</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>اباحه</td>\n",
              "      <td>'ebAhe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>اباحه‌گری</td>\n",
              "      <td>'ebAhegari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>اباحی</td>\n",
              "      <td>'ebAhi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>اباحیگر</td>\n",
              "      <td>'ebAhigar</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>اباحیگری</td>\n",
              "      <td>'ebAhigari</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>ابادت</td>\n",
              "      <td>'ebAdat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>اباذر</td>\n",
              "      <td>'abAzar</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>ابار</td>\n",
              "      <td>'abAr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>اباریق</td>\n",
              "      <td>'abAriq</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>اباطیل</td>\n",
              "      <td>'abAtil</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>اباعبدالرحمن</td>\n",
              "      <td>'abA'abdorrahmAn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>اباعبدالله</td>\n",
              "      <td>'abA'abdellAh</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>اباعبدالملک</td>\n",
              "      <td>'abA'abdolmalek</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>اباعد</td>\n",
              "      <td>'abA'ed</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>اباقا</td>\n",
              "      <td>'AbAqA</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>اباقاخان</td>\n",
              "      <td>'AbAqAxAn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>ابالسه</td>\n",
              "      <td>'abAlese</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>ابالیس</td>\n",
              "      <td>'abAlis</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>ابام</td>\n",
              "      <td>'abAm</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>ابان</td>\n",
              "      <td>'abAn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>ابانت</td>\n",
              "      <td>'ebAnat</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-78aaa0c1-65eb-4d0e-8260-b90dc38530cb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-78aaa0c1-65eb-4d0e-8260-b90dc38530cb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-78aaa0c1-65eb-4d0e-8260-b90dc38530cb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Splitting data to train,validation,test"
      ],
      "metadata": {
        "id": "OmqM_4ENxnid"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "graphemes=list(data[\"grapheme\"])\n",
        "phonemes=list(data[\"phoneme\"])\n",
        "len(graphemes),len(phonemes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yWiKfJLWw-j9",
        "outputId": "a7046a43-2e48-44f9-cce1-c9b05aac95f7"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(75162, 75162)"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_data=[]\n",
        "for i in range(len(graphemes)):\n",
        "  all_data.append((graphemes[i],phonemes[i]))"
      ],
      "metadata": {
        "id": "TCcKtJGSxe1k"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(all_data),all_data[70000]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xZGsj3AVx3Bn",
        "outputId": "22d2317d-d549-40f5-acba-44a8bfca4cfd"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(75162, ('نگاشتن', 'negAStan'))"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "metadata": {
        "id": "WNNHBzhfx4wf"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_iter,test_iter=train_test_split(all_data,test_size=0.2,random_state=42,shuffle=True)"
      ],
      "metadata": {
        "id": "agJ7cV6lzOTw"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(train_iter),len(test_iter)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "snc_SdNIzrKf",
        "outputId": "7e5957b2-fa63-49e7-8b2a-849f97506b36"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(60129, 15033)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizer:\n",
        "Since both graphemes and phonemes have no space character we just list the string for tokenizer"
      ],
      "metadata": {
        "id": "ENgi18T7z7YB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def my_tokenizer(word : str):\n",
        "  return list(word)"
      ],
      "metadata": {
        "id": "VZRJlGNrzzRS"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building vocab and token transform"
      ],
      "metadata": {
        "id": "kIV-Q01A0iiW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from typing import Iterable, List\n",
        "\n",
        "token_transform = {}\n",
        "vocab_transform = {}\n",
        "\n",
        "SRC_LANGUAGE = 'grapheme'\n",
        "TGT_LANGUAGE = 'phoneme'\n",
        "\n",
        "token_transform[SRC_LANGUAGE] = my_tokenizer\n",
        "token_transform[TGT_LANGUAGE] = my_tokenizer\n",
        "\n",
        "# yield token for build_vocab_from_iterator function\n",
        "def yield_tokens(data_iter: Iterable, language: str) -> List[str]:\n",
        "    language_index = {SRC_LANGUAGE: 0, TGT_LANGUAGE: 1}\n",
        "\n",
        "    for data_sample in data_iter:\n",
        "        yield token_transform[language](data_sample[language_index[language]])\n",
        "\n",
        "#special symbols and indices\n",
        "UNK_IDX, PAD_IDX, BOS_IDX, EOS_IDX = 0, 1, 2, 3\n",
        "special_symbols = ['<unk>', '<pad>', '<bos>', '<eos>']\n",
        "\n",
        "for ln in [SRC_LANGUAGE, TGT_LANGUAGE]:\n",
        "    # Training data Iterator\n",
        "    all_iter=all_data\n",
        "    # Create torchtext's Vocab object\n",
        "    vocab_transform[ln] = build_vocab_from_iterator(yield_tokens(all_data, ln),\n",
        "                                                    min_freq=1,\n",
        "                                                    specials=special_symbols,\n",
        "                                                    special_first=True)"
      ],
      "metadata": {
        "id": "1WrTkS5e0Nuy"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating Transformer Model"
      ],
      "metadata": {
        "id": "1SeiGMi72ds2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch import Tensor\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.nn import Transformer\n",
        "import math\n",
        "DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "metadata": {
        "id": "k5pQfLYx2Up9"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## PositionalEncoding"
      ],
      "metadata": {
        "id": "FCjFGhSn2oQK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# helper Module that adds positional encoding to the token embedding to introduce a notion of word order.\n",
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self,\n",
        "                 emb_size: int,\n",
        "                 dropout: float,\n",
        "                 maxlen: int = 5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        den = torch.exp(- torch.arange(0, emb_size, 2)* math.log(10000) / emb_size)\n",
        "        pos = torch.arange(0, maxlen).reshape(maxlen, 1)\n",
        "        pos_embedding = torch.zeros((maxlen, emb_size))\n",
        "        pos_embedding[:, 0::2] = torch.sin(pos * den)\n",
        "        pos_embedding[:, 1::2] = torch.cos(pos * den)\n",
        "        pos_embedding = pos_embedding.unsqueeze(-2)\n",
        "\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "        self.register_buffer('pos_embedding', pos_embedding)\n",
        "\n",
        "    def forward(self, token_embedding: Tensor):\n",
        "        return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0), :])\n"
      ],
      "metadata": {
        "id": "9GvjQ0zS2ikk"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## TokenEmbedding"
      ],
      "metadata": {
        "id": "rcWxTeAS2xF7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# helper Module to convert tensor of input indices into corresponding tensor of token embeddings\n",
        "class TokenEmbedding(nn.Module):\n",
        "    def __init__(self, vocab_size: int, emb_size):\n",
        "        super(TokenEmbedding, self).__init__()\n",
        "        self.embedding = nn.Embedding(vocab_size, emb_size)\n",
        "        self.emb_size = emb_size\n",
        "\n",
        "    def forward(self, tokens: Tensor):\n",
        "        return self.embedding(tokens.long()) * math.sqrt(self.emb_size)\n"
      ],
      "metadata": {
        "id": "6FiiR2912wfk"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Seq2Seq Transformer model"
      ],
      "metadata": {
        "id": "jTukhlxU26KP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Seq2Seq Network\n",
        "class Seq2SeqTransformer(nn.Module):\n",
        "    def __init__(self,\n",
        "                 num_encoder_layers: int,\n",
        "                 num_decoder_layers: int,\n",
        "                 emb_size: int,\n",
        "                 nhead: int,\n",
        "                 src_vocab_size: int,\n",
        "                 tgt_vocab_size: int,\n",
        "                 dim_feedforward: int = 512,\n",
        "                 dropout: float = 0.1):\n",
        "        super(Seq2SeqTransformer, self).__init__()\n",
        "        self.transformer = Transformer(d_model=emb_size,\n",
        "                                       nhead=nhead,\n",
        "                                       num_encoder_layers=num_encoder_layers,\n",
        "                                       num_decoder_layers=num_decoder_layers,\n",
        "                                       dim_feedforward=dim_feedforward,\n",
        "                                       dropout=dropout)\n",
        "        self.generator = nn.Linear(emb_size, tgt_vocab_size)\n",
        "        self.src_tok_emb = TokenEmbedding(src_vocab_size, emb_size)\n",
        "        self.tgt_tok_emb = TokenEmbedding(tgt_vocab_size, emb_size)\n",
        "        self.positional_encoding = PositionalEncoding(\n",
        "            emb_size, dropout=dropout)\n",
        "\n",
        "    def forward(self,\n",
        "                src: Tensor,\n",
        "                trg: Tensor,\n",
        "                src_mask: Tensor,\n",
        "                tgt_mask: Tensor,\n",
        "                src_padding_mask: Tensor,\n",
        "                tgt_padding_mask: Tensor,\n",
        "                memory_key_padding_mask: Tensor):\n",
        "        src_emb = self.positional_encoding(self.src_tok_emb(src))\n",
        "        tgt_emb = self.positional_encoding(self.tgt_tok_emb(trg))\n",
        "        outs = self.transformer(src_emb, tgt_emb, src_mask, tgt_mask, None,\n",
        "                                src_padding_mask, tgt_padding_mask, memory_key_padding_mask)\n",
        "        return self.generator(outs)\n",
        "\n",
        "    def encode(self, src: Tensor, src_mask: Tensor):\n",
        "        return self.transformer.encoder(self.positional_encoding(\n",
        "                            self.src_tok_emb(src)), src_mask)\n",
        "\n",
        "    def decode(self, tgt: Tensor, memory: Tensor, tgt_mask: Tensor):\n",
        "        return self.transformer.decoder(self.positional_encoding(\n",
        "                          self.tgt_tok_emb(tgt)), memory,\n",
        "                          tgt_mask)"
      ],
      "metadata": {
        "id": "d6T1y95v20Mj"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Masking mechanism\n",
        "During training, we need a subsequent character mask that will prevent model to look into the future characters when making predictions. We will also need masks to hide source and target padding tokens. Below, let’s define a function that will take care of both.\n",
        "\n",
        "Reference : https://pytorch.org/tutorials/beginner/translation_transformer.html"
      ],
      "metadata": {
        "id": "uhMLMRfP3Hjx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_square_subsequent_mask(sz):\n",
        "    mask = (torch.triu(torch.ones((sz, sz), device=DEVICE)) == 1).transpose(0, 1)\n",
        "    mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))\n",
        "    return mask\n",
        "\n",
        "\n",
        "def create_mask(src, tgt):\n",
        "    src_seq_len = src.shape[0]\n",
        "    tgt_seq_len = tgt.shape[0]\n",
        "\n",
        "    tgt_mask = generate_square_subsequent_mask(tgt_seq_len)\n",
        "    src_mask = torch.zeros((src_seq_len, src_seq_len),device=DEVICE).type(torch.bool)\n",
        "\n",
        "    src_padding_mask = (src == PAD_IDX).transpose(0, 1)\n",
        "    tgt_padding_mask = (tgt == PAD_IDX).transpose(0, 1)\n",
        "    return src_mask, tgt_mask, src_padding_mask, tgt_padding_mask"
      ],
      "metadata": {
        "id": "hMOIgPG93AW8"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building Model"
      ],
      "metadata": {
        "id": "lhYmlTCH3oLB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "torch.manual_seed(42)\n",
        "\n",
        "# Hyperparameters\n",
        "SRC_VOCAB_SIZE = len(vocab_transform[SRC_LANGUAGE])\n",
        "TGT_VOCAB_SIZE = len(vocab_transform[TGT_LANGUAGE])\n",
        "EMB_SIZE = 512\n",
        "NHEAD = 8\n",
        "FFN_HID_DIM = 512\n",
        "BATCH_SIZE = 128\n",
        "NUM_ENCODER_LAYERS = 5\n",
        "NUM_DECODER_LAYERS = 5\n",
        "\n",
        "# Instantiating Model\n",
        "g2p_model = Seq2SeqTransformer(NUM_ENCODER_LAYERS, NUM_DECODER_LAYERS, EMB_SIZE,\n",
        "                                 NHEAD, SRC_VOCAB_SIZE, TGT_VOCAB_SIZE, FFN_HID_DIM)\n",
        "\n",
        "# weights and biases initialization\n",
        "for p in g2p_model.parameters():\n",
        "    if p.dim() > 1:\n",
        "        nn.init.xavier_uniform_(p)\n",
        "\n",
        "g2p_model = g2p_model.to(DEVICE)\n",
        "\n",
        "# CrossEntropy Loss function\n",
        "loss_fn = torch.nn.CrossEntropyLoss(ignore_index=PAD_IDX)\n",
        "\n",
        "# Adam optimizer\n",
        "optimizer = torch.optim.Adam(g2p_model.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)"
      ],
      "metadata": {
        "id": "koVESPMY3kWX"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Collation\n",
        "our data iterator yields a pair of raw strings. We need to convert these string pairs into the batched tensors that can be processed by our Seq2Seq network defined previously. Below we define our collate function that convert batch of raw strings into batch tensors that can be fed directly into our model.\n",
        "\n",
        "Reference : https://pytorch.org/tutorials/beginner/translation_transformer.html"
      ],
      "metadata": {
        "id": "S6gMpVjS4yvf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.nn.utils.rnn import pad_sequence\n",
        "\n",
        "# helper function to club together sequential operations\n",
        "def sequential_transforms(*transforms):\n",
        "    def func(txt_input):\n",
        "        for transform in transforms:\n",
        "            txt_input = transform(txt_input)\n",
        "        return txt_input\n",
        "    return func\n",
        "\n",
        "# function to add BOS/EOS and create tensor for input sequence indices\n",
        "def tensor_transform(token_ids: List[int]):\n",
        "    return torch.cat((torch.tensor([BOS_IDX]),\n",
        "                      torch.tensor(token_ids),\n",
        "                      torch.tensor([EOS_IDX])))\n",
        "\n",
        "# src and tgt language text transforms to convert raw strings into tensors indices\n",
        "text_transform = {}\n",
        "for ln in [SRC_LANGUAGE, TGT_LANGUAGE]:\n",
        "    text_transform[ln] = sequential_transforms(token_transform[ln], #Tokenization\n",
        "                                               vocab_transform[ln], #Numericalization\n",
        "                                               tensor_transform) # Add BOS/EOS and create tensor\n",
        "\n",
        "\n",
        "# function to collate data samples into batch tesors\n",
        "def collate_fn(batch):\n",
        "    src_batch, tgt_batch = [], []\n",
        "    for src_sample, tgt_sample in batch:\n",
        "        src_batch.append(text_transform[SRC_LANGUAGE](src_sample.rstrip(\"\\n\")))\n",
        "        tgt_batch.append(text_transform[TGT_LANGUAGE](tgt_sample.rstrip(\"\\n\")))\n",
        "\n",
        "    src_batch = pad_sequence(src_batch, padding_value=PAD_IDX)\n",
        "    tgt_batch = pad_sequence(tgt_batch, padding_value=PAD_IDX)\n",
        "    return src_batch, tgt_batch"
      ],
      "metadata": {
        "id": "FUbvc6Sg4tb6"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Training"
      ],
      "metadata": {
        "id": "JWFor7mF50Rj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torch.utils.data import DataLoader"
      ],
      "metadata": {
        "id": "u6FASGE56CGZ"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training function"
      ],
      "metadata": {
        "id": "g2Q8x6k955wk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train_function(model, optimizer):\n",
        "    model.train()\n",
        "    losses = 0\n",
        "    train_dataloader = DataLoader(train_iter, batch_size=BATCH_SIZE, collate_fn=collate_fn)\n",
        "\n",
        "    for src, tgt in train_dataloader:\n",
        "        src = src.to(DEVICE)\n",
        "        tgt = tgt.to(DEVICE)\n",
        "\n",
        "        tgt_input = tgt[:-1, :]\n",
        "\n",
        "        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "\n",
        "        logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        tgt_out = tgt[1:, :]\n",
        "        loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1))\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "        losses += loss.item()\n",
        "\n",
        "    return losses / len(train_dataloader)"
      ],
      "metadata": {
        "id": "Yn6buel_5wKV"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluating function"
      ],
      "metadata": {
        "id": "7tu1KMvH6jSa"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate_function(model):\n",
        "    model.eval()\n",
        "    losses = 0\n",
        "\n",
        "    val_iter=test_iter\n",
        "    val_dataloader = DataLoader(val_iter, batch_size=BATCH_SIZE, collate_fn=collate_fn)\n",
        "\n",
        "    for src, tgt in val_dataloader:\n",
        "        src = src.to(DEVICE)\n",
        "        tgt = tgt.to(DEVICE)\n",
        "\n",
        "        tgt_input = tgt[:-1, :]\n",
        "\n",
        "        src_mask, tgt_mask, src_padding_mask, tgt_padding_mask = create_mask(src, tgt_input)\n",
        "\n",
        "        logits = model(src, tgt_input, src_mask, tgt_mask,src_padding_mask, tgt_padding_mask, src_padding_mask)\n",
        "\n",
        "        tgt_out = tgt[1:, :]\n",
        "        loss = loss_fn(logits.reshape(-1, logits.shape[-1]), tgt_out.reshape(-1))\n",
        "        losses += loss.item()\n",
        "\n",
        "    return losses / len(val_dataloader)"
      ],
      "metadata": {
        "id": "4Uq5tsC16irJ"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train model for N epochs"
      ],
      "metadata": {
        "id": "i0QR4D2e648V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from timeit import default_timer as timer\n",
        "NUM_EPOCHS = 20\n",
        "\n",
        "stats_for_plotting={\"train_losses\":[],\"test_losses\":[]}\n",
        "for epoch in range(1, NUM_EPOCHS+1):\n",
        "    start_time = timer()\n",
        "    train_loss = train_function(g2p_model, optimizer)\n",
        "    end_time = timer()\n",
        "    val_loss = evaluate_function(g2p_model)\n",
        "    print((f\"Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, \"f\"Epoch time = {(end_time - start_time):.3f}s\"))\n",
        "    stats_for_plotting[\"train_losses\"].append(train_loss)\n",
        "    stats_for_plotting[\"test_losses\"].append(val_loss)\n",
        "    torch.save(g2p_model.state_dict(),f\"./G2P/epoch_{epoch}.pth\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u6vXLjQh61Xw",
        "outputId": "0c0e39c9-4b6c-4ff5-94ed-8e857a8b7364"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1, Train loss: 1.433, Val loss: 0.644, Epoch time = 52.701s\n",
            "Epoch: 2, Train loss: 0.557, Val loss: 0.330, Epoch time = 51.212s\n",
            "Epoch: 3, Train loss: 0.366, Val loss: 0.259, Epoch time = 53.043s\n",
            "Epoch: 4, Train loss: 0.303, Val loss: 0.233, Epoch time = 52.922s\n",
            "Epoch: 5, Train loss: 0.266, Val loss: 0.224, Epoch time = 53.390s\n",
            "Epoch: 6, Train loss: 0.241, Val loss: 0.203, Epoch time = 53.591s\n",
            "Epoch: 7, Train loss: 0.220, Val loss: 0.189, Epoch time = 53.762s\n",
            "Epoch: 8, Train loss: 0.205, Val loss: 0.182, Epoch time = 53.812s\n",
            "Epoch: 9, Train loss: 0.191, Val loss: 0.173, Epoch time = 53.910s\n",
            "Epoch: 10, Train loss: 0.179, Val loss: 0.165, Epoch time = 53.947s\n",
            "Epoch: 11, Train loss: 0.170, Val loss: 0.161, Epoch time = 53.964s\n",
            "Epoch: 12, Train loss: 0.160, Val loss: 0.155, Epoch time = 53.987s\n",
            "Epoch: 13, Train loss: 0.153, Val loss: 0.153, Epoch time = 53.979s\n",
            "Epoch: 14, Train loss: 0.145, Val loss: 0.147, Epoch time = 54.013s\n",
            "Epoch: 15, Train loss: 0.139, Val loss: 0.144, Epoch time = 54.052s\n",
            "Epoch: 16, Train loss: 0.132, Val loss: 0.142, Epoch time = 53.968s\n",
            "Epoch: 17, Train loss: 0.127, Val loss: 0.140, Epoch time = 53.977s\n",
            "Epoch: 18, Train loss: 0.122, Val loss: 0.139, Epoch time = 53.982s\n",
            "Epoch: 19, Train loss: 0.117, Val loss: 0.137, Epoch time = 53.912s\n",
            "Epoch: 20, Train loss: 0.112, Val loss: 0.137, Epoch time = 53.963s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for param_group in optimizer.param_groups:\n",
        "        param_group['lr'] = lr=0.00005"
      ],
      "metadata": {
        "id": "FccES2ElYkC6"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(21, NUM_EPOCHS+10):\n",
        "    start_time = timer()\n",
        "    train_loss = train_function(g2p_model, optimizer)\n",
        "    end_time = timer()\n",
        "    val_loss = evaluate_function(g2p_model)\n",
        "    print((f\"Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, \"f\"Epoch time = {(end_time - start_time):.3f}s\"))\n",
        "    stats_for_plotting[\"train_losses\"].append(train_loss)\n",
        "    stats_for_plotting[\"test_losses\"].append(val_loss)\n",
        "    torch.save(g2p_model.state_dict(),f\"./G2P/epoch_{epoch}.pth\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LY7GVj6oYj6U",
        "outputId": "bf255a72-ed46-4688-f598-074a84233c1c"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 21, Train loss: 0.097, Val loss: 0.129, Epoch time = 51.935s\n",
            "Epoch: 22, Train loss: 0.091, Val loss: 0.128, Epoch time = 52.588s\n",
            "Epoch: 23, Train loss: 0.087, Val loss: 0.129, Epoch time = 53.207s\n",
            "Epoch: 24, Train loss: 0.083, Val loss: 0.129, Epoch time = 53.571s\n",
            "Epoch: 25, Train loss: 0.081, Val loss: 0.128, Epoch time = 53.811s\n",
            "Epoch: 26, Train loss: 0.078, Val loss: 0.129, Epoch time = 54.012s\n",
            "Epoch: 27, Train loss: 0.076, Val loss: 0.131, Epoch time = 54.278s\n",
            "Epoch: 28, Train loss: 0.074, Val loss: 0.129, Epoch time = 54.248s\n",
            "Epoch: 29, Train loss: 0.072, Val loss: 0.131, Epoch time = 54.338s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for param_group in optimizer.param_groups:\n",
        "        param_group['lr'] = lr=0.00001"
      ],
      "metadata": {
        "id": "mOMDDHnNa9a0"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(30, 32):\n",
        "    start_time = timer()\n",
        "    train_loss = train_function(g2p_model, optimizer)\n",
        "    end_time = timer()\n",
        "    val_loss = evaluate_function(g2p_model)\n",
        "    print((f\"Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, \"f\"Epoch time = {(end_time - start_time):.3f}s\"))\n",
        "    stats_for_plotting[\"train_losses\"].append(train_loss)\n",
        "    stats_for_plotting[\"test_losses\"].append(val_loss)\n",
        "    torch.save(g2p_model.state_dict(),f\"./G2P/epoch_{epoch}.pth\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mGzk5eYga9Vu",
        "outputId": "794569b1-dc44-4351-a307-b76755d95695"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 30, Train loss: 0.064, Val loss: 0.130, Epoch time = 52.731s\n",
            "Epoch: 31, Train loss: 0.060, Val loss: 0.130, Epoch time = 55.893s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(32, 36):\n",
        "    start_time = timer()\n",
        "    train_loss = train_function(g2p_model, optimizer)\n",
        "    end_time = timer()\n",
        "    val_loss = evaluate_function(g2p_model)\n",
        "    print((f\"Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, \"f\"Epoch time = {(end_time - start_time):.3f}s\"))\n",
        "    stats_for_plotting[\"train_losses\"].append(train_loss)\n",
        "    stats_for_plotting[\"test_losses\"].append(val_loss)\n",
        "    torch.save(g2p_model.state_dict(),f\"./G2P/epoch_{epoch}.pth\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JM_wMyp7i-sN",
        "outputId": "b90a9631-f43a-49f6-feec-bec8009c13bb"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 32, Train loss: 0.058, Val loss: 0.130, Epoch time = 50.871s\n",
            "Epoch: 33, Train loss: 0.058, Val loss: 0.130, Epoch time = 53.468s\n",
            "Epoch: 34, Train loss: 0.056, Val loss: 0.130, Epoch time = 54.544s\n",
            "Epoch: 35, Train loss: 0.056, Val loss: 0.131, Epoch time = 54.645s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for param_group in optimizer.param_groups:\n",
        "        param_group['lr'] = lr=0.000005"
      ],
      "metadata": {
        "id": "XnKSIxNmjnF1"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for epoch in range(36, 38):\n",
        "    start_time = timer()\n",
        "    train_loss = train_function(g2p_model, optimizer)\n",
        "    end_time = timer()\n",
        "    val_loss = evaluate_function(g2p_model)\n",
        "    print((f\"Epoch: {epoch}, Train loss: {train_loss:.3f}, Val loss: {val_loss:.3f}, \"f\"Epoch time = {(end_time - start_time):.3f}s\"))\n",
        "    stats_for_plotting[\"train_losses\"].append(train_loss)\n",
        "    stats_for_plotting[\"test_losses\"].append(val_loss)\n",
        "    torch.save(g2p_model.state_dict(),f\"./G2P/epoch_{epoch}.pth\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "18IjcIp5jqu4",
        "outputId": "984d2697-b8df-409b-93bd-7c6b6023c4f4"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 36, Train loss: 0.054, Val loss: 0.131, Epoch time = 54.643s\n",
            "Epoch: 37, Train loss: 0.053, Val loss: 0.131, Epoch time = 54.285s\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Plotting stats"
      ],
      "metadata": {
        "id": "26iDyq_lL61Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib import pyplot as plt"
      ],
      "metadata": {
        "id": "JntFHyhBMTDN"
      },
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.plot(stats_for_plotting[\"train_losses\"],label=\"train\")\n",
        "plt.plot(stats_for_plotting[\"test_losses\"],label=\"validation\")\n",
        "plt.legend(loc=\"upper right\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "G4NKYn7TL9xI",
        "outputId": "98e76450-aa61-4cc8-f9a0-1454260b0b4f"
      },
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xcZZ3n8c+vqqvv6XsngVxIiAFyJQmdiCAaRJzIjCjKLers4qrMMDIOO5eVcXfFcXTWmZfDsq5cFh1EdwWGgUEZFxYvE0AFkcQhIQmXJCQh93Qn6Uv6Xl2//eOc6q50qjsd6Orq7vN9v171qqpzTp369YH0t5/nOec55u6IiEh0xfJdgIiI5JeCQEQk4hQEIiIRpyAQEYk4BYGISMQV5LuA01VXV+dz5szJdxkiIhPKhg0bmty9Ptu6CRcEc+bMYf369fkuQ0RkQjGz3UOtU9eQiEjEKQhERCJOQSAiEnETboxARCaX3t5e9u7dS1dXV75LmRSKi4uZOXMmiURixJ9REIhIXu3du5cpU6YwZ84czCzf5Uxo7s6RI0fYu3cvc+fOHfHn1DUkInnV1dVFbW2tQmAUmBm1tbWn3bpSEIhI3ikERs9bOZaRCYLXDrbxjade42h7T75LEREZVyITBDub2vnWuu0caOnMdykiMo40Nzdz1113nfbnrrjiCpqbm3NQ0diLTBBUlgQj6C2dvXmuRETGk6GCIJlMDvu5J554gqqqqlyVNaZyFgRmdp+ZHTazzafYbqWZJc3s6lzVAhlB0KEgEJEBt956Kzt27GDZsmWsXLmSSy65hCuvvJKFCxcC8JGPfIQLLriARYsWce+99/Z/bs6cOTQ1NbFr1y4WLFjAZz/7WRYtWsQHPvABOjsnVs9DLk8fvR/4FvD9oTYwszjwt8BPclgHAJWlahGIjHd/9S9b2Lq/dVT3ufDMCm770KIh13/9619n8+bNvPTSSzz99NP87u/+Lps3b+4//fK+++6jpqaGzs5OVq5cycc+9jFqa2tP2Me2bdt48MEH+fa3v821117Lo48+yic/+clR/TlyKWctAnd/Fjh6is3+GHgUOJyrOtKq1DUkIiOwatWqE87B/+Y3v8n555/PhRdeyJ49e9i2bdtJn5k7dy7Lli0D4IILLmDXrl1jVe6oyNsFZWY2A7gKuBRYeYptbwRuBJg9e/Zb+r7SwjgFMVMQiIxjw/3lPlbKysr6Xz/99NP87Gc/4/nnn6e0tJTVq1dnPUe/qKio/3U8Hp9wXUP5HCy+A/iCu6dOtaG73+vuDe7eUF+fdTrtUzIzKksSNCsIRCTDlClTaGtry7qupaWF6upqSktLefXVV/n1r389xtWNjXxOMdEAPBRe/FAHXGFmSXf/Ya6+sLIkoRaBiJygtraWiy++mMWLF1NSUsK0adP6161Zs4Z77rmHBQsWcO6553LhhRfmsdLcyVsQuHt/J5yZ3Q/8OJchAMGAcauCQEQGeeCBB7IuLyoq4sknn8y6Lj0OUFdXx+bNAydH/vmf//mo15drOQsCM3sQWA3Umdle4DYgAeDu9+Tqe4dTWZLQlcUiIoPkLAjcfe1pbHtDrurIVFmSYGdT+1h8lYjIhBGZK4shCIJmXVAmInKCyAVBa1cvqZTnuxQRkXEjckHgDm3dw88hIiISJZELAkBnDomIZIhkEGicQETeqvLycgD279/P1Vdnnytz9erVrF+/ftj93HHHHXR0dPS/z+e01pEMAl1UJiJv15lnnskjjzzylj8/OAjyOa11pIKgqrQQUBCIyIBbb72VO++8s//9l7/8Zb761a9y2WWXsWLFCpYsWcKPfvSjkz63a9cuFi9eDEBnZyfXX389CxYs4KqrrjphrqGbbrqJhoYGFi1axG233QYEE9nt37+fSy+9lEsvvRQYmNYa4Pbbb2fx4sUsXryYO+64o//7cjXddT6nmBhzahGIjHNP3goHXx7dfU5fAh/8+pCrr7vuOm655RY+97nPAfDwww/z1FNP8fnPf56Kigqampq48MILufLKK4e8H/Ddd99NaWkpr7zyCps2bWLFihX96772ta9RU1NDX18fl112GZs2beLzn/88t99+O+vWraOuru6EfW3YsIHvfve7vPDCC7g773znO3nve99LdXV1zqa7jlSLoH+MoFNXF4tIYPny5Rw+fJj9+/ezceNGqqurmT59Ol/84hdZunQp73//+9m3bx+HDh0ach/PPvts/y/kpUuXsnTp0v51Dz/8MCtWrGD58uVs2bKFrVu3DlvPL3/5S6666irKysooLy/nox/9KL/4xS+A3E13HakWQXEiRmE8phaByHg1zF/uuXTNNdfwyCOPcPDgQa677jp+8IMf0NjYyIYNG0gkEsyZMyfr9NOnsnPnTr7xjW/w4osvUl1dzQ033PCW9pOWq+muI9UiMDNNPCciJ7nuuut46KGHeOSRR7jmmmtoaWlh6tSpJBIJ1q1bx+7du4f9/Hve857+ies2b97Mpk2bAGhtbaWsrIzKykoOHTp0wgR2Q01/fckll/DDH/6Qjo4O2tvbeeyxx7jkkktG8ac9WaRaBKCpqEXkZIsWLaKtrY0ZM2Zwxhln8IlPfIIPfehDLFmyhIaGBs4777xhP3/TTTfxqU99igULFrBgwQIuuOACAM4//3yWL1/Oeeedx6xZs7j44ov7P3PjjTeyZs0azjzzTNatW9e/fMWKFdxwww2sWrUKgM985jMsX748p3c9M/eJNd1CQ0ODn+r83OF87O7nKCqI8cBnJ+e84iITzSuvvMKCBQvyXcakku2YmtkGd2/Itn2kuoZALQIRkcEiFwRVCgIRkRNELggqFAQi485E66Iez97KsYxcEFSWJGjrStKnqahFxoXi4mKOHDmiMBgF7s6RI0coLi4+rc9F8qwhCGYgrS4rzHM1IjJz5kz27t1LY2NjvkuZFIqLi5k5c+ZpfSZyQVBVOjDNhIJAJP8SiQRz587NdxmRFsmuIdB8QyIiaTkLAjO7z8wOm9nmIdZ/wsw2mdnLZvacmZ2fq1oyDcw3pCAQEYHctgjuB9YMs34n8F53XwL8NXBvDmvppxaBiMiJcjZG4O7PmtmcYdY/l/H218DpjW68RZWlCgIRkUzjZYzg08CTQ600sxvNbL2ZrX+7ZxbovsUiIifKexCY2aUEQfCFobZx93vdvcHdG+rr69/W9xUVxClOaCpqEZG0vJ4+amZLge8AH3T3I2P1vZUlCZo7dHMaERHIY4vAzGYD/wz8vru/PpbfXVVSqBaBiEgoZy0CM3sQWA3Umdle4DYgAeDu9wBfAmqBu8L7gCaHmiJ1tGkGUhGRAbk8a2jtKdZ/BvhMrr5/OBUlCfY1j84t3kREJrq8DxbnQ2VJghaNEYiIABENgqpSdQ2JiKRFMggqSxK09/TR25fKdykiInkX2SAAXVQmIgIRDwJNPCciEtUg0HxDIiL9ohkEmoFURKRfpINAYwQiIhEPguYOBYGISKSDQF1DIiIRDYJEPEZZYVxBICJCRIMANPGciEhaZIOgoiShMQIRESIcBFWlCZ01JCJChINAXUMiIgEFgYhIxEU6CJo7dU8CEZHIBkFVaSFdvSm6k335LkVEJK8iGwQVuqhMRATIYRCY2X1mdtjMNg+x3szsm2a23cw2mdmKXNWSjeYbEhEJ5LJFcD+wZpj1HwTmh48bgbtzWMtJNM2EiEggZ0Hg7s8CR4fZ5MPA9z3wa6DKzM7IVT2DaeI5EZFAPscIZgB7Mt7vDZedxMxuNLP1Zra+sbFxVL68Si0CERFgggwWu/u97t7g7g319fWjsk91DYmIBPIZBPuAWRnvZ4bLxoTOGhIRCeQzCB4H/l149tCFQIu7HxirL4/HjClFBRojEJHIK8jVjs3sQWA1UGdme4HbgASAu98DPAFcAWwHOoBP5aqWoVRq4jkRkdwFgbuvPcV6Bz6Xq+8fCc03JCIyQQaLc0VBICKiIKBZQSAiERfpIKgqVYtARCTSQVChriERkWgHQWVJgp5kiq5eTUUtItEV+SAAzTckItEW6SCoKikEdHWxiERbpINA8w2JiCgIAAWBiESbggBo7tBN7EUkuqIdBKVqEYiIRDoIphQVYKb7FotItEU6CGIxo6JYF5WJSLRFOghA8w2JiEQ+CDTfkIhEXeSDQFNRi0jURT4INPGciERd5IOgsiRBi+YaEpEIi3wQVIUtguDOmSIi0RP5IKgsSZBMOR09mopaRKIpp0FgZmvM7DUz225mt2ZZP9vM1pnZv5nZJjO7Ipf1ZKP5hkQk6nIWBGYWB+4EPggsBNaa2cJBm/0X4GF3Xw5cD9yVq3qGoiAQkajLZYtgFbDd3d9w9x7gIeDDg7ZxoCJ8XQnsz2E9WaXnG9LNaUQkqkYUBGZWZmax8PU5ZnalmSVO8bEZwJ6M93vDZZm+DHzSzPYCTwB/PMT332hm681sfWNj40hKHjG1CEQk6kbaIngWKDazGcBPgN8H7h+F718L3O/uM4ErgP+dDpxM7n6vuze4e0N9ff0ofO2AdBBo4jkRiaqRBoG5ewfwUeAud78GWHSKz+wDZmW8nxkuy/Rp4GEAd38eKAbqRljTqFCLQESibsRBYGbvAj4B/N9wWfwUn3kRmG9mc82skGAw+PFB27wJXBZ+wQKCIBjdvp9TKC8qIB4zmjt1cxoRiaaRBsEtwF8Cj7n7FjM7G1g33AfcPQncDDwFvEJwdtAWM/uKmV0ZbvZnwGfNbCPwIHCDj/GVXWam+YZEJNIKRrKRuz8DPAMQ9uE3ufvnR/C5JwgGgTOXfSnj9Vbg4tMpOBeCIEjmuwwRkbwY6VlDD5hZhZmVAZuBrWb2F7ktbexo4jkRibKRdg0tdPdW4CPAk8BcgjOHJoWqkgQtuoG9iETUSIMgEV438BHgcXfvJbgYbFLQGIGIRNlIg+B/AbuAMuBZMzsLaM1VUWNNQSAiUTaiIHD3b7r7DHe/wgO7gUtzXNuYSQdBKjVpGjkiIiM20sHiSjO7PT3Ng5n9PUHrYFKoKk2QcjjeozOHRCR6Rto1dB/QBlwbPlqB7+aqqLFWkb66WBPPiUgEjeg6AmCeu38s4/1fmdlLuSgoHzKnmZh1im1FRCabkbYIOs3s3ek3ZnYx0JmbksaeJp4TkSgbaYvgD4Hvm1ll+P4Y8O9zU9LYSwdBs4JARCJopFNMbATON7OK8H2rmd0CbMplcWOlqlQzkIpIdJ3WHcrcvTW8whjgT3NQT15oKmoRibK3c6tKG7Uq8qwkEScRNwWBiETS2wmCSXP1VXoqat23WESiaNgxAjNrI/svfANKclJRnlSWJHTWkIhE0rBB4O5TxqqQnGt8HbY8BhfdDIUnXxSt+YZEJKreTtfQxHJkGzz9N3Boa9bVCgIRiaroBMH0JcHzwexnvCoIRCSqohMElbOguAoOvpx1dVVpIc26OY2IRFBOg8DM1pjZa2a23cxuHWKba81sq5ltMbMHclhM0CoYIggqShK0dSc1FbWIRE7OgsDM4sCdwAeBhcBaM1s4aJv5wF8CF7v7IuCWXNUDwPSlcGgLpPpOWlVZksAd2ro0FbWIREsuWwSrgO3u/oa79wAPAR8etM1ngTvd/RiAux/OYT1BiyDZCUd2nLRKVxeLSFTlMghmAHsy3u8Nl2U6BzjHzH5lZr82szU5rGfYAeOq/onnNE4gItGS78HiAmA+sBpYC3zbzKoGb2RmN6bvjtbY2PjWv63uHIgXZh0nqNTEcyISUbkMgn1wwn1eZobLMu0FHnf3XnffCbxOEAwncPd73b3B3Rvq6+vfekUFhVB/XvYgUNeQiERULoPgRWC+mc01s0LgeuDxQdv8kKA1gJnVEXQVvZHDmoIBYwWBiEi/nAWBuyeBm4GngFeAh919i5l9xcyuDDd7CjhiZluBdcBfuPuRXNUEBOME7Yeh7dAJi/tvTqOJ50QkYkZ6h7K3xN2fAJ4YtOxLGa+d4L4GY3dvg/4B45dhyrT+xcWJOEUFMU08JyKRk+/B4rE3fXHwfHDjSauqShM0HddZQyISLdELguJKqDor6zjBsllVPLejiaChIiISDdELAhhyqonLF07nQEsXm/e1ZvmQiMjkFNEgWBpcXdx9/ITF7ztvKjGDn249mKfCRETGXkSDYAngcPjEexPUlBXSMKeGn2w9lP1zIiKTUISDgKxTTXxg4TRePdjGnqMdY1yUiEh+RDMIKmcOeW+CyxcGp5SqVSAiURHNIDCDM7JfYXxWbRnnTCvXOIGIREY0gwAG7k3Qd/L9By5fOI0Xdx3THctEJBIiHARLINkFR0++N8EHFk6nL+X866u5vT2CiMh4EO0ggKzdQ0tmVDKtooifapxARCIgukHQf2+Ck88cisWM9y+YxjOvN9LVe/JtLUVEJpPoBkE8AVMXDHkz+8sXTqOjp4/nd+R2MlQRkXyLbhBA0D10YBNkmVvoXfNqKS8q0GmkIjLpRTwIlkJHExw/+Zd9UUGc955Tz89eOUQqpUnoRGTyingQhAPGB04eJ4Cge6ixrZuX9jaPYVEiImMr2kEwbVHwnGXAGODSc6cSj5nOHhKRSS3aQVBcCdVzhhwwrixN8M65NQoCEZnUoh0EMOS9CdIuXziN7YePs7OpfQyLEhEZOwqC6Uvh6BvQ3ZZ1dXoSOs09JCKTVU6DwMzWmNlrZrbdzG4dZruPmZmbWUMu68kqfW+CQ1uzrp5ZXcqCMyrUPSQik1bOgsDM4sCdwAeBhcBaM1uYZbspwJ8AL+SqlmENc2+CtMsXTmPD7mMcOd49RkWJiIydXLYIVgHb3f0Nd+8BHgI+nGW7vwb+FujKYS1Dq5gBJdXDjhN8YOE0Ug4/1yR0IjIJ5TIIZgB7Mt7vDZf1M7MVwCx3/7/D7cjMbjSz9Wa2vrGxcXSrNAvGCYYJgkVnVnBmZbG6h0RkUsrbYLGZxYDbgT871bbufq+7N7h7Q319/egXM31JcP/iLPcmADAzLl84jV9sa6SzR5PQicjkkssg2AfMyng/M1yWNgVYDDxtZruAC4HH8zNgvDS4N8GR7UNucvnC6XT1pvjFtlFukYiI5Fkug+BFYL6ZzTWzQuB64PH0Sndvcfc6d5/j7nOAXwNXuvv6HNaU3TD3Jkh759k1TCkuUPeQiEw6OQsCd08CNwNPAa8AD7v7FjP7ipldmavvfUvq5kO8aNgzhxLxGJeeO5Wfv3pY3UMiMqnkdIzA3Z9w93PcfZ67fy1c9iV3fzzLtqvz0hqAU96bIO26lbM41tHDH/1gAz3J1BgVJyKSW7qyOC091USWexOkXfyOOr72kSWse62RP/unjfRpemoRmQQUBGnpexO0HRh2s4+/czZfWHMe/7JxP//1R5vxYYJDRGQiKMh3AeNG5oBxxZnDbnrT6nm0dPZyzzM7qCpJ8J/WnDcGBYqI5IaCIC19b4IDm+Cc3znl5l9Ycy6tXb3c9fQOKksS/MF75+W4QBGR3FDXUFpxBUxbAuv/AVqH7x6C4CKzv/7wYn5v6Rn8tydf5cHfvDkGRYqIjD4FQaar7oGuVnjo49DbecrN4zHj9muXcem59XzxsZf58ab9Y1CkiMjoUhBkmr4YPvYd2P9v8MM/GvYMorTCghh3feICVp5Vw3/8x5d4+jVNTCciE4uCYLDzroD33wZb/hme+bsRfaSkMM53bmjg3OlT+MP/s4Gfv6Krj0Vk4lAQZHPxLXD+Wnj6b2DLYyP6SEVxgu99ahVzasv49PfW87kHfsvh1vzMrC0icjoUBNmYwe/dATNXwWM3BV1FI1BbXsSPbr6YP7v8HH669RCX/f0zfP/5XbrwTETGNQXBUBLFcP0PoKwOHvw4tI3snsVFBXH++LL5/OSW97BsdhVf+tEWPnr3c2zZ35LjgkVE3hoFwXDKp8LaB6GrBR5cO6IzidLm1JXx/f+wiv9x/TL2Hevgym/9iq/+eCvt3dnveSAiki8KglOZvgQ+ei/s/y386HMjOpMozcz48LIZ/PxPV3Pdyll855c7ef/tz/DEywdIqbtIRMYJBcFILPg9uOxLsPlReOZvTysMACpLE/zNVUt49KaLqCxJ8Ec/+C3v+/unue+XO2nr6s1R0SIiI2MTbdK0hoYGX78+D7NVu8NjfwCb/hGmLYYLb4LFVwdjCaehty/F/9t8kPuf28WG3ccoK4xzTcMs/t27zuLs+vIcFS8iUWdmG9w96x0gFQSno68XNj4Ev74bDm+Bsnpo+DSs/HQwnnCaNu1t5v7ndvHjjQfo6Uux+tx6brhoDu+ZX08sZjn4AUQkqhQEo80ddj4Dz98F256CeCEsuTZoJUxffNq7a2zr5oEX3uT/vLCbxrZuzq4r4+qGmXxo6ZnMqinNwQ8gIlGjIMilpm1BC2Hjg9DbAXPfA+d8EOrOCW6BWTkLYiMbiulJpnhy8wG+//xuNuw+BsCK2VVcef6Z/O7SM6mfUpTLn0REJjEFwVjoOAq//R68+A/QsmdgeUEx1M4PQqFufhAQ05dC/TnD7m7P0Q7+ZdN+Hn9pP68ebCNmcNG8Oq48/0x+Z/F0KksSOf6BRGQyURCMJXfoOAJNr4ePbQOvj+0GwuM9dREsvSYYcK6aNewutx1q4/GN+3l84352H+mgMB7j3fPruCR8zKsvx0xjCiIytLwFgZmtAf4HEAe+4+5fH7T+T4HPAEmgEfgP7r57uH2O+yAYTm8XHN0Bu34FL/8T7P1NsHz2RbDkalh0FZTWDPlxd2fT3hYe37ifn71yiN1HOgCYXlHMu+fX8e531HHxO+rUhSQiJ8lLEJhZHHgduBzYC7wIrHX3rRnbXAq84O4dZnYTsNrdrxtuvxM6CAY7uhM2PwKb/gmaXoNYAt7x/iAU5r1v2FCAoPvol9ub+OW2Jn61o4nmjuCahPOmT+GS+XWsnFPDirOqqStXMIhEXb6C4F3Al939d8L3fwng7v9tiO2XA99y94uH2++kCoI0dzi4KWglvPwotIU3uKmdD7NWwcyVwXP9eRCLZ91FX8rZsr+FX2wLgmHD7mP09KUAmFNbyoqzqrkgfMyfOoW4Tk8ViZR8BcHVwBp3/0z4/veBd7r7zUNs/y3goLt/Ncu6G4EbAWbPnn3B7t3D9h5NbKk+2PMbePM52PNi0H3UcSRYV1QBMy4IQuGMZVA1O3gUV5y0m67ePjbva2HD7mNs2H2M3755jKbjPQBMKSpg2ewqls+u5vyZlSyZWcnUKad3YZyITCzjPgjM7JPAzcB73b17uP1OyhbBcNzh6BtBOOz9TRAOh7eApwa2Ka4KBpyrzgpOV00HRN18qDkb4gncnTePdvQHw4bdx3j9UBvpKY+mVxSzZGZlGAxVLJlRSU1ZYX5+ZhEZdcMFQUEOv3cfkHk6zMxw2QnM7P3Af2YEIRBJZlA7L3gsWxss626Dxteh5U1oTj/2wJEdsGMd9LYPfD5WADXzsPpzOav+XM6qP4+PXngO/N5KOjzB1v2tbNzbwst7m9m0r4Wfbh24u9qMqhLOmVbOO6YGj3n1wXNVqQJCZDLJZYuggGCw+DKCAHgR+Li7b8nYZjnwCEHLYdtI9hu5FsHpcofOY3BsZ3DqauNrwaPptaBl0d+SsKAVUT0HqudCzVyonsPxslls6azhpcPOy/ta2H74OG80tdOTHGiB1JUXMq++nHlTy3lH+nlqOWdUFGtqDJFxKp+nj14B3EFw+uh97v41M/sKsN7dHzeznwFLgAPhR9509yuH26eC4G1IdgethsZXg3A4uiM4c+nYLuhoOnHbkuqgq6mkCi+qoJ1SjvUVcbinkINdCfa0x9nZFmdPTykHvJYDXoMlSjm7vqy/9TCvvpyz68uYXVNKWVEuG58iciq6oExOras1CIRju4LWxLFdQZdTVyt0twbdUekH2f+f6YhX0BirY29fDW/0VHHAaznk1bRTTKKohClTKqiqrKCmspK6qkrqa6s5o7aKM6dNo7hI3U0iuZSvMQKZSIor4IylwWM4qVQwBtHdFoREeyO07oOWvZS27uOsln2c1bqfi1o3YJ3HBj7nQGv42HPiLpMe45BV0VpQR1fxVFLl04hXnEFp7Qwqp86mun4GsaIySJQMPApKRjyHk4gMT0EgpycWg6IpwaPiTOC8rJsZQE87HD8EPR2Q7Apu9dnbSaq3g7a2NppbW2htbaWrpRFvO0hh5yHK2vdS07aR2oNtpyylL1aIFwTBEC8sxTKDIjMwEiWQKA2fizNel564TbwQ4onwURgMtPcvC58LSiCufzYyuej/aMmdwrLg9NVBYkBl+MimJ5li15FmDu9/k2OH3uT40QO0Hz9OZ8dxujqO09t1nHhfNyXWQ3FPNyX0UGw9VBQkqSxIUh5vpswOU2K9FNFNItVNQV8XluzEvO/t/1wWDwOkKAiGgqKBIIFgQN5TwcC992W8T0FheTAwX3N28KgOX0+ZHpwhJpKN+8CJHkNcVPp2KAhk3CksiDFnWg1zptUAy05a7+60diY52NrFgZZODrZ0sbO1i4MtXRxoCZ73t3TS1pU86bMVhc6MMjiz1Jle5kwtdqaWpKgr7KO6GKqKgkdFwim0vuBmRH09kEoGg+3Jbkh2Bs+9nSe/NwuCwmLhw4J/uOn3nc1wYCNsfTwIibREaXAGV9VZQbDE4kGLxOJBK8ziA8tgIFhSfRmhkxoInlRfxussyy0W7CtWMLDfzPcWD3/xeMa+feB78YGfubczOAa9XcFU7OnWXyoZhmXxoOd0iBZlHNeu7M+pZEaLLGyVxRIntt7IDNAs41cnHCsfdCzCdWbBfsyCY4MN/PdLvx+WDxzbVDJ8pAZe9/9BkFGj+8DxTe9j8HFOH+u0i2+By//qFLWcPgWBTDhmRmVpgsrSBOdOnzLkdse7kxwMg+FgaxdNx7tpbOum6XjweLGlm6Z9PRxt78n6+fKiAuqnFFFXXkj9lCJqy4qoLiukpjRBdVkh1aWF1JQVhssKKSk8jb/U+nqD6cqP7gxO6z26Mxikb94TBE//L5RUxuvwOf3Lqf8xKHgsNvDLPDOE+l/HCX5xJU/c7wnv+078nsxfkulfmgXFA11tpXVZxnDiwc+S+cu9t2vgfVdL2N1WHJyldrbeQioAAAr4SURBVFJgFAffl0oG++nrGQjmvt6B1yf/DzJ4wYk/u2V5DwO/hE/6hTzCE2piGeFq8YyQzfhvka6nv0Y7sd4T/rvGOOm/9axVI/wf7PQoCGTSKi8q6L8Ybji9fSmOtvfQ2NZNY0ZYNLYNvH7tYBtNx4/Q2tU75O+FooIYNWWFJzyqSwupDcOitqyQ2bWlLJheQSyeGOge4rLR/+FFToOCQCIvEY8xraKYaRWnnm+pL+W0dPZytL2HYx1Ba6K5o4ej7b0cbe/mWEew7mh7D28e7eDo8R7auk/soqoqTfCus2u5aF4t75pXx7z6Mt1PQvJKQSByGuIx6/9rf6R6kimOdfRw5HgPrx1q5bntR3huxxGe3HwQgKlTirhoXi0XzavjnWfXMLO6VLPDypjSBWUieeDu7DnayXM7mvjVjiM8v6Opf3bYeMyoLy9iemUx0yuKg+eM13XlhUwpTlBRnKA4EVNrQkZEF5SJjDNmxuzaUmbXzub6VbNxd7YdPs6G3cfY39zJgZYuDrV2sb3xOL/a3nRS91JaIm5hKBRQUZJgSnEBFcUJqkoLqS5NUFWafp1+HzyXFRVQVKAQkYCCQGQcMDPOmTaFc6ZlPwsq8wyoox09tHb20taVpLWrl9bOXlq7krSFrw+2dNHS2cuxjl76UkO3+M2CAe7iRJzigjjFieB1ell5UQFl4aO8KB4+DywrLohRWBCjMB4+F8RIxAeWFSVilCTiFCfiJOK6Cnw8UxCITAAjPQMqk7vT1p2kub2X5s4ejnX00tzRw7H2Htp7+uju7aMrmQqee1N0JfvoCl939vZxsLWL9u4kx7v7aO9O0tn71i/GK4hZEAqF8TAcgpAoygieooIYRWEgpZ8rSxLMrinlrNoyzqrV5IW5oqMqMkmZGRXhWMJsSt/2/vpSTntPMgiHriTdyRTdyRS9fSl6kuGjL3ifXtfd20dnTx+dGQHTFT46evroTvZxvDtJ0/Eeunv76E6m6Mp4Tg5q0dSVF54QDLNrSqkpK6SiJEFlSfCzVpYkKCxQC+R0KAhEZETisYFgGXJ+kFHW2tXLm0c62H2kg11H2oPXR9t54Y0jPPZvJ93nql+6NVFZkqC0sIDCgljY4jixO6uoIE5hQYx4zIiZEY9B3IxYzAaeY0ZB+EgUxEjEYhTEjUQ8RiJuFITv4xmfKYgN7CO978ICC1s6A91w46XLTEEgIuNWRXGCxTMqWTzj5OTp6u1jX3MnzR09tHYmaenspbWrl5aO8Lmzl9bOJO09QeulrSvJkbDV0pNM0Z3s62/J9LmTSkGf+7DjKqMtHjOKw66xwoIYMbPgYmIjeE3wTPj++pWz+MwlJ8/f9XYpCERkQipOxJlXP/Ixk9ORSnl/KKTcSaacZJ+T7AuCJNnnJFMpepLBc29fsF1fykmlgu2DcBnYR9BVFgRQV2/Q9dWV8bonmSLl4Hgwu4V7+B5SHryoKy/Kyc+rIBARGSQWM2IYidGf6HNcGh8dVCIikjcKAhGRiMtpEJjZGjN7zcy2m9mtWdYXmdk/hutfMLM5uaxHREROlrMgMLM4cCfwQWAhsNbMFg7a7NPAMXd/B/Dfgb/NVT0iIpJdLlsEq4Dt7v6Gu/cADwEfHrTNh4Hvha8fAS4zTX4iIjKmchkEM4A9Ge/3hsuybuPuSaAFqM1hTSIiMsiEGCw2sxvNbL2ZrW9sbMx3OSIik0oug2AfMCvj/cxwWdZtzKyA4ML1I4N35O73unuDuzfU19fnqFwRkWjK5QVlLwLzzWwuwS/864GPD9rmceDfA88DVwP/6qe4U86GDRuazGz3W6ypDmh6i58daxOlVtU5+iZKrapzdOW6zrOGWpGzIHD3pJndDDwFxIH73H2LmX0FWO/ujwP/APxvM9sOHCUIi1Pt9y03Ccxs/VB36BlvJkqtqnP0TZRaVefoymedOZ1iwt2fAJ4YtOxLGa+7gGtyWYOIiAxvQgwWi4hI7kQtCO7NdwGnYaLUqjpH30SpVXWOrrzVaacYmxURkUkuai0CEREZREEgIhJxkQmCU82EOl6Y2S4ze9nMXjKz9fmuJ5OZ3Wdmh81sc8ayGjP7qZltC5+r81ljWFO2Or9sZvvC4/qSmV2RzxrDmmaZ2Toz22pmW8zsT8Ll4+qYDlPneDymxWb2GzPbGNb6V+HyueEMx9vDGY8Lx2md95vZzoxjumxM6onCGEE4E+rrwOUEcx69CKx19615LSwLM9sFNLj7uLsAxszeAxwHvu/ui8NlfwccdfevhwFb7e5fGId1fhk47u7fyGdtmczsDOAMd/+tmU0BNgAfAW5gHB3TYeq8lvF3TA0oc/fjZpYAfgn8CfCnwD+7+0Nmdg+w0d3vHod1/iHwY3d/ZCzriUqLYCQzocopuPuzBBf+ZcqcQfZ7BL8g8mqIOscddz/g7r8NX7cBrxBMxDiujukwdY47Hjgevk2EDwfeRzDDMYyPYzpUnXkRlSAYyUyo44UDPzGzDWZ2Y76LGYFp7n4gfH0QmJbPYk7hZjPbFHYd5b0LK1N4U6blwAuM42M6qE4Yh8fUzOJm9hJwGPgpsANoDmc4hnHy739wne6ePqZfC4/pfzez3NytfpCoBMFE8m53X0FwQ5/Phd0cE0I4T9R47Wu8G5gHLAMOAH+f33IGmFk58Chwi7u3Zq4bT8c0S53j8pi6e5+7LyOY6HIVcF6eS8pqcJ1mthj4S4J6VwI1wJh0CUYlCEYyE+q44O77wufDwGME/yOPZ4fCPuR0X/LhPNeTlbsfCv/hpYBvM06Oa9g//CjwA3f/53DxuDum2eocr8c0zd2bgXXAu4CqcIZjGGf//jPqXBN2w7m7dwPfZYyOaVSCoH8m1PBsgesJZj4dV8ysLByMw8zKgA8Am4f/VN6lZ5AlfP5RHmsZUvoXa+gqxsFxDQcM/wF4xd1vz1g1ro7pUHWO02Nab2ZV4esSghNEXiH4RXt1uNl4OKbZ6nw14w8AIxjHGJNjGomzhgDCU9vuYGAm1K/luaSTmNnZBK0ACCYEfGA81WlmDwKrCabLPQTcBvwQeBiYDewGrnX3vA7UDlHnaoIuDAd2AX+Q0Q+fF2b2buAXwMtAKlz8RYL+93FzTIepcy3j75guJRgMjhP8ofuwu38l/Lf1EEF3y78Bnwz/6h5vdf4rUA8Y8BLwhxmDyrmrJypBICIi2UWla0hERIagIBARiTgFgYhIxCkIREQiTkEgIhJxCgKRQcysL2P2x5dsFGerNbM5ljErqsh4kNOb14tMUJ3hpf8ikaAWgcgIWXCviL+z4H4RvzGzd4TL55jZv4YThf3czGaHy6eZ2WPhnPMbzeyicFdxM/t2OA/9T8IrS0XyRkEgcrKSQV1D12Wsa3H3JcC3CK5UB/ifwPfcfSnwA+Cb4fJvAs+4+/nACmBLuHw+cKe7LwKagY/l+OcRGZauLBYZxMyOu3t5luW7gPe5+xvhJGwH3b3WzJoIbtzSGy4/4O51ZtYIzMycyiCcxvmn7j4/fP8FIOHuX839TyaSnVoEIqfHh3h9OjLnuOlDY3WSZwoCkdNzXcbz8+Hr5whmtAX4BMEEbQA/B26C/puQVI5VkSKnQ3+JiJysJLxzVNr/c/f0KaTVZraJ4K/6teGyPwa+a2Z/ATQCnwqX/wlwr5l9muAv/5sIbuAiMq5ojEBkhMIxggZ3b8p3LSKjSV1DIiIRpxaBiEjEqUUgIhJxCgIRkYhTEIiIRJyCQEQk4hQEIiIR9/8BNDH3//Vn24AAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install jiwer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iPU63r-eGzkc",
        "outputId": "72ae7f78-5f3e-426b-abab-c63f7396a879"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting jiwer\n",
            "  Downloading jiwer-2.5.1-py3-none-any.whl (15 kB)\n",
            "Collecting levenshtein==0.20.2\n",
            "  Downloading Levenshtein-0.20.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4 MB 37.0 MB/s \n",
            "\u001b[?25hCollecting rapidfuzz<3.0.0,>=2.3.0\n",
            "  Downloading rapidfuzz-2.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0 MB 64.8 MB/s \n",
            "\u001b[?25hCollecting jarowinkler<2.0.0,>=1.2.0\n",
            "  Downloading jarowinkler-1.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (104 kB)\n",
            "\u001b[K     |████████████████████████████████| 104 kB 78.3 MB/s \n",
            "\u001b[?25hInstalling collected packages: jarowinkler, rapidfuzz, levenshtein, jiwer\n",
            "Successfully installed jarowinkler-1.2.1 jiwer-2.5.1 levenshtein-0.20.2 rapidfuzz-2.8.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inference"
      ],
      "metadata": {
        "id": "0nKRNNr37A7F"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## decoder function\n",
        "it is called greedy decoder because we don't do any beam search for results"
      ],
      "metadata": {
        "id": "kPfQE_Hv7LV_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# function to generate output sequence\n",
        "def greedy_decode(model, src, src_mask, max_len, start_symbol):\n",
        "    src = src.to(DEVICE)\n",
        "    src_mask = src_mask.to(DEVICE)\n",
        "\n",
        "    memory = model.encode(src, src_mask)\n",
        "    ys = torch.ones(1, 1).fill_(start_symbol).type(torch.long).to(DEVICE)\n",
        "    for i in range(max_len-1):\n",
        "        memory = memory.to(DEVICE)\n",
        "        tgt_mask = (generate_square_subsequent_mask(ys.size(0))\n",
        "                    .type(torch.bool)).to(DEVICE)\n",
        "        out = model.decode(ys, memory, tgt_mask)\n",
        "        out = out.transpose(0, 1)\n",
        "        prob = model.generator(out[:, -1])\n",
        "        _, next_word = torch.max(prob, dim=1)\n",
        "        next_word = next_word.item()\n",
        "\n",
        "        ys = torch.cat([ys,\n",
        "                        torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=0)\n",
        "        if next_word == EOS_IDX:\n",
        "            break\n",
        "    return ys\n"
      ],
      "metadata": {
        "id": "z7LXAnFH7CCQ"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Inference function\n",
        "gives a grapheme and outputs the phoneme"
      ],
      "metadata": {
        "id": "IxGW_FeW7iOc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def inference(model: torch.nn.Module, src_sentence: str):\n",
        "    model.eval()\n",
        "    src = text_transform[SRC_LANGUAGE](src_sentence).view(-1, 1)\n",
        "    num_tokens = src.shape[0]\n",
        "    src_mask = (torch.zeros(num_tokens, num_tokens)).type(torch.bool)\n",
        "    tgt_tokens = greedy_decode(\n",
        "        model,  src, src_mask, max_len=num_tokens + 5, start_symbol=BOS_IDX).flatten()\n",
        "    return \" \".join(vocab_transform[TGT_LANGUAGE].lookup_tokens(list(tgt_tokens.cpu().numpy()))).replace(\"<bos>\", \"\").replace(\"<eos>\", \"\")"
      ],
      "metadata": {
        "id": "l1CcrIQE7c1E"
      },
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Evaluating model last checkpoint"
      ],
      "metadata": {
        "id": "iUYHHLFURoyb"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1.Saving all predictions"
      ],
      "metadata": {
        "id": "XFgpPuo2RvCW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "all_preds=[]\n",
        "for gs,ps in tqdm(test_iter):\n",
        "  ground_truth=ps\n",
        "  myoutput=inference(g2p_model,gs)\n",
        "  all_preds.append(myoutput.replace(\" \",\"\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h-ro0MFg-LQI",
        "outputId": "13d06b46-4150-4f94-827f-f6a491521012"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 15033/15033 [16:09<00:00, 15.51it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2.Calculating WER and PER\n",
        "WER : Word error rate\n",
        "\n",
        "PER : Phoneme error rate"
      ],
      "metadata": {
        "id": "9ErrVOkfR3cL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from jiwer import cer,wer"
      ],
      "metadata": {
        "id": "T2JbrYTGG_Gh"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trues=0\n",
        "all_num=0\n",
        "average_cer=0\n",
        "average_wer=0\n",
        "for i in range(len(all_preds)):\n",
        "  if all_preds[i]==test_iter[i][1]:\n",
        "    trues+=1\n",
        "  all_num+=1\n",
        "  average_cer+=cer(test_iter[i][1],all_preds[i])\n",
        "  average_wer+=wer(test_iter[i][1],all_preds[i])\n",
        "average_cer/=len(all_preds)\n",
        "average_wer/=len(all_preds)\n",
        "exact_accuracy=trues/all_num\n",
        "print(f\"Exact Accuracy : {exact_accuracy}\")\n",
        "print(f\"PER : {average_cer}\")\n",
        "print(f\"WER : {average_wer}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2xZhmT9M-LJJ",
        "outputId": "92fc3447-9d73-4ee4-d8e7-9049204e3841"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Exact Accuracy : 0.7345839153861504\n",
            "PER : 0.061936786813605815\n",
            "WER : 0.2654160846138495\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Some examples:\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "C6aO6KBM8G9K"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "some_example_df=pd.DataFrame(columns=['Grapheme','Predicted Phoneme','Correct Phoneme'])"
      ],
      "metadata": {
        "id": "EjQbOo10hDac"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "some_examples=random.sample(test_iter, 20)\n",
        "example_gs=[]\n",
        "example_gt_phonemes=[]\n",
        "example_pred_phonemes=[]\n",
        "for example in some_examples:\n",
        "  example_gs.append(example[0])\n",
        "  example_gt_phonemes.append(example[1])\n",
        "  example_pred_phonemes.append(inference(g2p_model,example[0]).replace(\" \",\"\"))\n",
        "some_example_df[\"Grapheme\"]=example_gs\n",
        "some_example_df[\"Predicted Phoneme\"]=example_pred_phonemes\n",
        "some_example_df[\"Correct Phoneme\"]=example_gt_phonemes\n",
        "some_example_df"
      ],
      "metadata": {
        "id": "xC1cACQr72dK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 676
        },
        "outputId": "ef001116-031a-41d7-ccf2-0b21b90c9a0b"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       Grapheme Predicted Phoneme Correct Phoneme\n",
              "0          تلمذ          talammoz        talammoz\n",
              "1     ناآمرزیده       nA'Amorzide     nA'Amorzide\n",
              "2     سایه‌پسند        sAyepasand      sAyepasand\n",
              "3       پرمایون          parmAyon        pormAyun\n",
              "4   مولوی‌پژوهی      molavipaZuhi    molavipaZuhi\n",
              "5   ولونتاریستی      voluntAristi    voluntAristi\n",
              "6     عادی‌سازی          'AdisAzi       'AddisAzi\n",
              "7          هرول             harul          hervol\n",
              "8     پیازکاران        piyAzkArAn      piyAzkArAn\n",
              "9       استظهار         'estezhAr       'estezhAr\n",
              "10      اورانوس           'urAnus         'orAnus\n",
              "11       سبغانه           sabqAne         sabqAne\n",
              "12          فخر              faxr            faxr\n",
              "13          یقر              yaqr           yoqor\n",
              "14      دفاعیات        defA'iyyAt      defA'iyyAt\n",
              "15        متمنی         motemanni       motemanni\n",
              "16        تانول             tAnul           tAnul\n",
              "17      تخته‌گل          taxtegol        taxtegel\n",
              "18      اسفراین         'esferAyn      'esfarAyen\n",
              "19           خا                xA              xA"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-92425d3b-9106-4a6b-ae13-8f297ee5b1d9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Grapheme</th>\n",
              "      <th>Predicted Phoneme</th>\n",
              "      <th>Correct Phoneme</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>تلمذ</td>\n",
              "      <td>talammoz</td>\n",
              "      <td>talammoz</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ناآمرزیده</td>\n",
              "      <td>nA'Amorzide</td>\n",
              "      <td>nA'Amorzide</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>سایه‌پسند</td>\n",
              "      <td>sAyepasand</td>\n",
              "      <td>sAyepasand</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>پرمایون</td>\n",
              "      <td>parmAyon</td>\n",
              "      <td>pormAyun</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>مولوی‌پژوهی</td>\n",
              "      <td>molavipaZuhi</td>\n",
              "      <td>molavipaZuhi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>ولونتاریستی</td>\n",
              "      <td>voluntAristi</td>\n",
              "      <td>voluntAristi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>عادی‌سازی</td>\n",
              "      <td>'AdisAzi</td>\n",
              "      <td>'AddisAzi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>هرول</td>\n",
              "      <td>harul</td>\n",
              "      <td>hervol</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>پیازکاران</td>\n",
              "      <td>piyAzkArAn</td>\n",
              "      <td>piyAzkArAn</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>استظهار</td>\n",
              "      <td>'estezhAr</td>\n",
              "      <td>'estezhAr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>اورانوس</td>\n",
              "      <td>'urAnus</td>\n",
              "      <td>'orAnus</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>سبغانه</td>\n",
              "      <td>sabqAne</td>\n",
              "      <td>sabqAne</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>فخر</td>\n",
              "      <td>faxr</td>\n",
              "      <td>faxr</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>یقر</td>\n",
              "      <td>yaqr</td>\n",
              "      <td>yoqor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>دفاعیات</td>\n",
              "      <td>defA'iyyAt</td>\n",
              "      <td>defA'iyyAt</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>متمنی</td>\n",
              "      <td>motemanni</td>\n",
              "      <td>motemanni</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>تانول</td>\n",
              "      <td>tAnul</td>\n",
              "      <td>tAnul</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>تخته‌گل</td>\n",
              "      <td>taxtegol</td>\n",
              "      <td>taxtegel</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>اسفراین</td>\n",
              "      <td>'esferAyn</td>\n",
              "      <td>'esfarAyen</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>خا</td>\n",
              "      <td>xA</td>\n",
              "      <td>xA</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-92425d3b-9106-4a6b-ae13-8f297ee5b1d9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-92425d3b-9106-4a6b-ae13-8f297ee5b1d9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-92425d3b-9106-4a6b-ae13-8f297ee5b1d9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Custom grapheme"
      ],
      "metadata": {
        "id": "Yc9ccPuApXpo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "inference(g2p_model,\"سلام\").replace(\" \",\"\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "kytNq8vig1oV",
        "outputId": "f8c030a5-55bd-4e18-e4a7-598e9f82a4b6"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'salAm'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "tDJN6f7CpivX"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}