Building Process.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "cc35d391",
      "metadata": {
        "id": "cc35d391"
      },
      "source": [
        "### Including Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0eb88f66",
      "metadata": {
        "id": "0eb88f66"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import spacy\n",
        "import codecs, sys\n",
        "import random\n",
        "unlp = spacy.blank('ur')\n",
        "from collections import Counter"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "981eb438",
      "metadata": {
        "id": "981eb438"
      },
      "source": [
        "### File Loading"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qt9TQefXOF6D",
        "outputId": "bccb658a-6d70-44d3-c2a8-ab2fc0cc5b33"
      },
      "id": "Qt9TQefXOF6D",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0f80d05b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0f80d05b",
        "outputId": "bb79dee8-7b26-4eb2-d4ca-cca5a853af12"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "        Unnamed: 0                                               Text\n",
              "0                0  aalmi bank askriat pasandi se mutasirah khanda...\n",
              "1                1  maali saal 2020 return file karne walon ki tad...\n",
              "2                2  japan ko sindh ke khusoosi iqtisadi zone mein ...\n",
              "3                3  bramdat 767 feesad barh kar arab 16 crore doll...\n",
              "4                4  ke electric ko izafi bijli gas ki farahmi ke q...\n",
              "...            ...                                                ...\n",
              "130316       18597  amrici fouj ne security khadshaat ke paish e n...\n",
              "130317       18598  duniya bhar mein masnoaat ki bar se le kar un ...\n",
              "130318       18599  krstina coach ne musalsal mah khalaa mein guza...\n",
              "130319       18600  apple ne apni smart watch mein 2018 mein e si ...\n",
              "130320       18601  huawei ka naya flag ship phone pi 40 is compan...\n",
              "\n",
              "[130321 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-893ee2c8-432e-42d1-87b8-5aa6ca71b910\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>aalmi bank askriat pasandi se mutasirah khanda...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>maali saal 2020 return file karne walon ki tad...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>japan ko sindh ke khusoosi iqtisadi zone mein ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>bramdat 767 feesad barh kar arab 16 crore doll...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>ke electric ko izafi bijli gas ki farahmi ke q...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130316</th>\n",
              "      <td>18597</td>\n",
              "      <td>amrici fouj ne security khadshaat ke paish e n...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130317</th>\n",
              "      <td>18598</td>\n",
              "      <td>duniya bhar mein masnoaat ki bar se le kar un ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130318</th>\n",
              "      <td>18599</td>\n",
              "      <td>krstina coach ne musalsal mah khalaa mein guza...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130319</th>\n",
              "      <td>18600</td>\n",
              "      <td>apple ne apni smart watch mein 2018 mein e si ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130320</th>\n",
              "      <td>18601</td>\n",
              "      <td>huawei ka naya flag ship phone pi 40 is compan...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>130321 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-893ee2c8-432e-42d1-87b8-5aa6ca71b910')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-893ee2c8-432e-42d1-87b8-5aa6ca71b910 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-893ee2c8-432e-42d1-87b8-5aa6ca71b910');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "df = pd.read_csv(\"/content/drive/MyDrive/Internship/Project 4/Urdu To Roman/Dataset.csv\")\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c7bbb087",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c7bbb087",
        "outputId": "32aaf5e0-98b9-4578-8e60-31185f8b7f27"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "130321 <class 'str'>\n"
          ]
        }
      ],
      "source": [
        "data = []\n",
        "\n",
        "for i in df['Text']:\n",
        "  data.append(i)\n",
        "\n",
        "print(len(data) , type(data[0]))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c262a7aa",
      "metadata": {
        "id": "c262a7aa"
      },
      "source": [
        "### Lower Casing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ac0abb20",
      "metadata": {
        "id": "ac0abb20"
      },
      "outputs": [],
      "source": [
        "for i in range(len(data)):\n",
        "    if type(data[i]) == 'str':\n",
        "        data[i] = data[i].lower()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "fea40e25",
      "metadata": {
        "id": "fea40e25"
      },
      "source": [
        "### Pre_Processing + Tokenizing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5ac8456d",
      "metadata": {
        "id": "5ac8456d"
      },
      "outputs": [],
      "source": [
        "tokens = []\n",
        "punc = '''!()%\\n٪-;۔،:\\n\\/'\"\\,“./؟_ء'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "63f9651d",
      "metadata": {
        "id": "63f9651d"
      },
      "outputs": [],
      "source": [
        "for sentence in data:\n",
        "\n",
        "    if type(sentence) == str: \n",
        "\n",
        "          sentence = unlp(sentence)\n",
        "          for word in sentence:\n",
        "              \n",
        "              word = str(word)\n",
        "              for index in word:\n",
        "                  \n",
        "                  if index in punc:\n",
        "                      word = word.replace(index,'')\n",
        "                      \n",
        "              if word != \"\\n\" and word != \"\\r\" and word != \" \" and word != '' and word != \" \\r\" and word !='‘' and word !='’':\n",
        "                      tokens.append(word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e5943247",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e5943247",
        "outputId": "5799cff7-2dc9-4f9e-e88f-41c4c3c9ba65"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8611713"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "len(tokens)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "cc289fe1",
      "metadata": {
        "id": "cc289fe1"
      },
      "source": [
        "### Bi-Gram Formation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "29675101",
      "metadata": {
        "id": "29675101"
      },
      "outputs": [],
      "source": [
        "bi_grams = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e0b894a3",
      "metadata": {
        "id": "e0b894a3"
      },
      "outputs": [],
      "source": [
        "for i in range(len(tokens)-1):\n",
        "\n",
        "    bi = tokens[i] + ' ' + tokens[i+1]\n",
        "    bi_grams.append(bi)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9b19bf02",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9b19bf02",
        "outputId": "ae3e0017-0e24-48aa-b5c0-9f3286923485"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8611712"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "len(bi_grams)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "ee64e8ad",
      "metadata": {
        "id": "ee64e8ad"
      },
      "source": [
        "### Tri-Gram Formation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4a9c55d0",
      "metadata": {
        "id": "4a9c55d0"
      },
      "outputs": [],
      "source": [
        "tri_grams = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b9fdd80d",
      "metadata": {
        "id": "b9fdd80d"
      },
      "outputs": [],
      "source": [
        "for i in range(len(tokens)-2):\n",
        "\n",
        "    tri = tokens[i] + ' ' + tokens[i+1] + ' ' + tokens[i+2]\n",
        "    tri_grams.append(tri)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "35128747",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "35128747",
        "outputId": "6e7562ee-d97e-4275-ff34-a07e6bbdeffc"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8611711"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "len(tri_grams)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8eaf99e9",
      "metadata": {
        "id": "8eaf99e9"
      },
      "source": [
        "### Frequency Dictionaries"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "edb01f7f",
      "metadata": {
        "id": "edb01f7f"
      },
      "source": [
        "Uni-gram"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "325895cf",
      "metadata": {
        "id": "325895cf"
      },
      "outputs": [],
      "source": [
        "uni_grams_freq = Counter(tokens)\n",
        "uni_grams_dictionary = dict(uni_grams_freq)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1c122d81",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1c122d81",
        "outputId": "4040519f-d6c2-4288-b9b4-410d4922d9fd"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "37218"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ],
      "source": [
        "len(uni_grams_dictionary.keys())"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "49789a22",
      "metadata": {
        "id": "49789a22"
      },
      "source": [
        "Bi-gram"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "97b9e600",
      "metadata": {
        "id": "97b9e600"
      },
      "outputs": [],
      "source": [
        "bi_grams_freq = Counter(bi_grams)\n",
        "bi_grams_dictionary = dict(bi_grams_freq)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bb2c4b2a",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bb2c4b2a",
        "outputId": "227b1d04-e497-4da6-f92e-ec087c6d84d7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1087623"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "len(bi_grams_dictionary.keys())"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5e198a26",
      "metadata": {
        "id": "5e198a26"
      },
      "source": [
        "Tri-gram"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "64379f4e",
      "metadata": {
        "id": "64379f4e"
      },
      "outputs": [],
      "source": [
        "tri_grams_freq = Counter(tri_grams)\n",
        "tri_grams_dictionary = dict(tri_grams_freq)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bebf065b",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bebf065b",
        "outputId": "8ffa8ba2-00d1-45c9-c476-6b1851556b1b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3534682"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "len(tri_grams_dictionary.keys())"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Dictionary Method (Approach 1)"
      ],
      "metadata": {
        "id": "HvSWNCxtj4nd"
      },
      "id": "HvSWNCxtj4nd"
    },
    {
      "cell_type": "code",
      "source": [
        "bi_gram_tokens = []\n",
        "\n",
        "for i in bi_grams_dictionary.keys():\n",
        "\n",
        "            # Default Parameters\n",
        "            N = len(uni_grams_freq.keys()) \n",
        "            count_min = 5 \n",
        "            thresh = 10 \n",
        "\n",
        "            tokens = i.split(' ')\n",
        "            a = tokens[0]\n",
        "            b = tokens[1]\n",
        "\n",
        "            # Accesing Uni-Gram Dictionary\n",
        "            try:\n",
        "                count_a = uni_grams_dictionary[a]\n",
        "            except:\n",
        "                count_a = 0\n",
        "\n",
        "            try:\n",
        "                count_b = uni_grams_dictionary[b]\n",
        "            except:\n",
        "                count_b = 0\n",
        "\n",
        "            # Accesing Bi-Gram Dictionary\n",
        "            try: \n",
        "                count_ab = bi_grams_dictionary[a + ' ' + b]\n",
        "            except:\n",
        "                count_ab = 0\n",
        "\n",
        "\n",
        "            # Applying Formula From Reserach Paper\n",
        "            numerator = count_ab - count_min\n",
        "            denominator = count_a * count_b\n",
        "            \n",
        "            division = (numerator + 1) / (denominator + 1) \n",
        "            result = division * N\n",
        "\n",
        "            if result > thresh:\n",
        "                bi_gram_tokens.append(a + ' ' + b)  "
      ],
      "metadata": {
        "id": "bVuyoSiTgRMn"
      },
      "id": "bVuyoSiTgRMn",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(bi_gram_tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cBmj5qG5rZf8",
        "outputId": "87a337a6-99b0-4e86-f66b-b01be4ea01b2"
      },
      "id": "cBmj5qG5rZf8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4974"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "bi_gram_tokens[30:40]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_vZkmaplhd2t",
        "outputId": "1a98c340-7de3-4f43-8fe0-556456db89f2"
      },
      "id": "_vZkmaplhd2t",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['omra zaireen',\n",
              " 'shibli frazz',\n",
              " 'gardishi qarzon',\n",
              " 'saam sang',\n",
              " 'chal basey',\n",
              " 'supreme court',\n",
              " 'giray list',\n",
              " 'zimma daari',\n",
              " 'with holding',\n",
              " 'mausam sarma']"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tri_gram_tokens = []\n",
        "\n",
        "for i in tri_grams_dictionary.keys():\n",
        "    \n",
        "        # Default Parameters\n",
        "        N = len(uni_grams_freq.keys()) \n",
        "        count_min = 5 \n",
        "        thresh = 10 \n",
        "\n",
        "        tokens = i.split(' ')\n",
        "        a = tokens[0]\n",
        "        b = tokens[1]\n",
        "        c = tokens[2]\n",
        "\n",
        "        try:\n",
        "            count_a = uni_grams_dictionary[a]\n",
        "        except:\n",
        "            count_a = 0\n",
        "\n",
        "        try:\n",
        "            count_b = uni_grams_dictionary[b]\n",
        "        except:\n",
        "            count_b = 0\n",
        "\n",
        "        try:\n",
        "            count_c = uni_grams_dictionary[c]\n",
        "        except:\n",
        "            count_c = 0\n",
        "\n",
        "\n",
        "        try: \n",
        "            count_abc = tri_grams_dictionary[a + ' ' + b + ' ' + c]\n",
        "        except:\n",
        "            count_abc = 0\n",
        "\n",
        "      # Applying formula\n",
        "        numerator = count_abc - count_min\n",
        "        denominator = count_a * count_b * count_c\n",
        "        \n",
        "        division = (numerator + 1) / (denominator + 1) \n",
        "        result = division * N\n",
        "        \n",
        "        if result > thresh:\n",
        "            tri_gram_tokens.append(a + ' ' + b + ' ' + c)"
      ],
      "metadata": {
        "id": "1NgRpwDkiCoM"
      },
      "id": "1NgRpwDkiCoM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tri_gram_tokens"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FflaxR7giDcO",
        "outputId": "bcf19607-a11b-4152-8c0e-c14a443c2d48"
      },
      "id": "FflaxR7giDcO",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['gloria vendor blt',\n",
              " 'saya khudaye zuljilal',\n",
              " 'muttahida quami movement',\n",
              " 'saghir ellahi khachi',\n",
              " 'kai po chey',\n",
              " 'prat bres dlas',\n",
              " 'bres dlas hvord',\n",
              " 'loshya ayonz giraye',\n",
              " 'atal bahari vajpayee',\n",
              " 'adolah dabir almulk']"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens = []\n",
        "tokens = bi_gram_tokens + tri_gram_tokens\n",
        "\n",
        "tokens_df = pd.DataFrame(tokens)\n",
        "tokens_df "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "9ECwHDANi1PF",
        "outputId": "807c32e7-7faf-4fad-c6b9-87c1678cf99c"
      },
      "id": "9ECwHDANi1PF",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                         0\n",
              "0          askriat pasandi\n",
              "1            khanay peenay\n",
              "2          aqwam mutahidda\n",
              "3                bar waqat\n",
              "4              ands motors\n",
              "...                    ...\n",
              "4979        prat bres dlas\n",
              "4980       bres dlas hvord\n",
              "4981   loshya ayonz giraye\n",
              "4982  atal bahari vajpayee\n",
              "4983   adolah dabir almulk\n",
              "\n",
              "[4984 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c87dfeb7-cac8-4457-b5ff-793e7d1581d0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>askriat pasandi</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>khanay peenay</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>aqwam mutahidda</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>bar waqat</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ands motors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4979</th>\n",
              "      <td>prat bres dlas</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4980</th>\n",
              "      <td>bres dlas hvord</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4981</th>\n",
              "      <td>loshya ayonz giraye</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4982</th>\n",
              "      <td>atal bahari vajpayee</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4983</th>\n",
              "      <td>adolah dabir almulk</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4984 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c87dfeb7-cac8-4457-b5ff-793e7d1581d0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c87dfeb7-cac8-4457-b5ff-793e7d1581d0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c87dfeb7-cac8-4457-b5ff-793e7d1581d0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokens_df.to_pickle(\"/content/drive/MyDrive/Internship/Project 4/model.pkl\")"
      ],
      "metadata": {
        "id": "PnT2pU1jrHkM"
      },
      "id": "PnT2pU1jrHkM",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('/content/drive/MyDrive/Internship/Project 4/model2.pkl', 'wb') as f:\n",
        "      pickle.dump(tokens, f)"
      ],
      "metadata": {
        "id": "2yh5l8VVtdEU"
      },
      "id": "2yh5l8VVtdEU",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "id": "24da4756",
      "metadata": {
        "id": "24da4756"
      },
      "source": [
        "### Roman Tokenizer (Approach 2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e9c03a5c",
      "metadata": {
        "id": "e9c03a5c"
      },
      "outputs": [],
      "source": [
        "def roman_tokenizer(sentence):\n",
        "   \n",
        "    # Lowering\n",
        "    sentence = sentence.lower()\n",
        "    # Pre Processing\n",
        "    tokens = []\n",
        "    sentence = unlp(sentence)\n",
        "    for word in sentence:\n",
        "\n",
        "        word = str(word)\n",
        "        for index in word:\n",
        "            \n",
        "              if index in punc:\n",
        "                    word = word.replace(index,'')\n",
        "\n",
        "        if word != \"\\n\" and word != \"\\r\" and word != \" \" and word != '' and word != \" \\r\" and word !='‘' and word !='’':\n",
        "              tokens.append(word)\n",
        "\n",
        "    # Default Parameters\n",
        "    N = len(uni_grams_freq.keys()) \n",
        "    count_min = 5 \n",
        "    thresh = 10 \n",
        "    \n",
        "    # For By Tokens\n",
        "    bi_tokens = [] \n",
        "    for i in range(len(tokens)):\n",
        "        \n",
        "        if i + 1 < len(tokens):\n",
        "\n",
        "            a = tokens[i]\n",
        "            b = tokens[i + 1]\n",
        "\n",
        "            # Accesing Uni-Gram Dictionary\n",
        "            try:\n",
        "                count_a = uni_grams_dictionary[a]\n",
        "            except:\n",
        "                count_a = 0\n",
        "\n",
        "            try:\n",
        "                count_b = uni_grams_dictionary[b]\n",
        "            except:\n",
        "                count_b = 0\n",
        "\n",
        "            # Accesing Bi-Gram Dictionary\n",
        "            try: \n",
        "                count_ab = bi_grams_dictionary[a + ' ' + b]\n",
        "            except:\n",
        "                count_ab = 0\n",
        "\n",
        "\n",
        "            # Applying Formula From Reserach Paper\n",
        "            numerator = count_ab - count_min\n",
        "            denominator = count_a * count_b\n",
        "            \n",
        "            division = (numerator + 1) / (denominator + 1) \n",
        "            result = division * N\n",
        "\n",
        "            if result > thresh:\n",
        "                bi_tokens.append(a + ' ' + b)\n",
        "                \n",
        "    for bitoken in bi_tokens:\n",
        "        separate = bitoken.split(' ')\n",
        "        \n",
        "        index = tokens.index(separate[0])\n",
        "        tokens.remove(separate[0])\n",
        "        tokens.remove(separate[1])\n",
        "        tokens.insert(index, bitoken)\n",
        "\n",
        "    # tri_tokens = []\n",
        "    # for i in range(len(tokens)):\n",
        "        \n",
        "    #     if i + 2 < len(tokens):\n",
        "\n",
        "    #         a = tokens[i]\n",
        "    #         b = tokens[i + 1]\n",
        "    #         c = tokens[i + 2]\n",
        "\n",
        "    #         try:\n",
        "    #             count_a = uni_grams_dictionary[a]\n",
        "    #         except:\n",
        "    #             count_a = 0\n",
        "\n",
        "    #         try:\n",
        "    #             count_b = uni_grams_dictionary[b]\n",
        "    #         except:\n",
        "    #             count_b = 0\n",
        "\n",
        "    #         try:\n",
        "    #             count_c = uni_grams_dictionary[c]\n",
        "    #         except:\n",
        "    #             count_c = 0\n",
        "\n",
        "\n",
        "    #         try: \n",
        "    #             count_abc = tri_grams_dictionary[a + ' ' + b + ' ' + c]\n",
        "    #         except:\n",
        "    #             count_abc = 0\n",
        "\n",
        "    #       # Applying formula\n",
        "    #         numerator = count_abc - count_min\n",
        "    #         denominator = count_a * count_b * count_c\n",
        "            \n",
        "    #         division = (numerator + 1) / (denominator + 1) \n",
        "    #         result = division * N\n",
        "            \n",
        "    #         if result > thresh:\n",
        "    #             tri_tokens.append(a + ' ' + b + ' ' + c)\n",
        "    \n",
        "    # Inserting Bi-Tokens In Tokenized array\n",
        "    \n",
        "    # for tritoken in tri_tokens:\n",
        "    #     separate = tritoken.split(' ')\n",
        "        \n",
        "    #     index = tokens.index(separate[0])\n",
        "    #     tokens.remove(separate[0])\n",
        "    #     tokens.remove(separate[1])\n",
        "    #     tokens.remove(separate[1])\n",
        "    #     tokens.insert(index, bitoken)\n",
        "\n",
        "    return tokens"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8ae2fc26",
      "metadata": {
        "id": "8ae2fc26"
      },
      "source": [
        "### Input"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "70fcef43",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "70fcef43",
        "outputId": "73f575e0-5805-4693-a7d2-9a37711ebc7b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Enter the text to Tokkenize: nawaz sharif kon hai?\n",
            "\n",
            "\n",
            "The tokenized string is as follow : \n",
            " ['nawaz sharif', 'kon', 'hai', '?']\n"
          ]
        }
      ],
      "source": [
        "sentence = input(\"Enter the text to Tokkenize: \")\n",
        "\n",
        "print(f\"\\n\\nThe tokenized string is as follow : \\n {roman_tokenizer(sentence)}\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = input(\"Enter the text to Tokkenize: \")\n",
        "\n",
        "print(f\"\\n\\nThe tokenized string is as follow : \\n {roman_tokenizer(sentence)}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NpmA3BQDQrVE",
        "outputId": "2c1370b3-365d-40ea-9a0b-f2f648034801"
      },
      "id": "NpmA3BQDQrVE",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Enter the text to Tokkenize: kab hoga aqwam mutahidda ka ijlas !\n",
            "\n",
            "\n",
            "The tokenized string is as follow : \n",
            " ['kab', 'hoga', 'aqwam mutahidda', 'ka', 'ijlas']\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "cc35d391",
        "981eb438",
        "c262a7aa",
        "fea40e25",
        "cc289fe1",
        "ee64e8ad",
        "24da4756",
        "8ae2fc26"
      ]
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}