IndicTrans_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "IndicTrans Model.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjfzxXZLHed_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "487f2bd3-7732-4fbc-ef22-636fc63a9f74"
      },
      "source": [
        "# clone the repo for running evaluation\n",
        "!git clone https://github.com/AI4Bharat/indicTrans.git\n",
        "%cd indicTrans\n",
        "# clone requirements repositories\n",
        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_library.git\n",
        "!git clone https://github.com/anoopkunchukuttan/indic_nlp_resources.git\n",
        "!git clone https://github.com/rsennrich/subword-nmt.git\n",
        "%cd .."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'indicTrans'...\n",
            "remote: Enumerating objects: 503, done.\u001b[K\n",
            "remote: Counting objects: 100% (206/206), done.\u001b[K\n",
            "remote: Compressing objects: 100% (87/87), done.\u001b[K\n",
            "remote: Total 503 (delta 163), reused 132 (delta 118), pack-reused 297\u001b[K\n",
            "Receiving objects: 100% (503/503), 1.49 MiB | 6.90 MiB/s, done.\n",
            "Resolving deltas: 100% (290/290), done.\n",
            "/content/indicTrans\n",
            "Cloning into 'indic_nlp_library'...\n",
            "remote: Enumerating objects: 1325, done.\u001b[K\n",
            "remote: Counting objects: 100% (147/147), done.\u001b[K\n",
            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
            "remote: Total 1325 (delta 84), reused 89 (delta 41), pack-reused 1178\u001b[K\n",
            "Receiving objects: 100% (1325/1325), 9.57 MiB | 11.37 MiB/s, done.\n",
            "Resolving deltas: 100% (688/688), done.\n",
            "Cloning into 'indic_nlp_resources'...\n",
            "remote: Enumerating objects: 139, done.\u001b[K\n",
            "remote: Counting objects: 100% (13/13), done.\u001b[K\n",
            "remote: Compressing objects: 100% (13/13), done.\u001b[K\n",
            "remote: Total 139 (delta 2), reused 2 (delta 0), pack-reused 126\u001b[K\n",
            "Receiving objects: 100% (139/139), 149.77 MiB | 29.76 MiB/s, done.\n",
            "Resolving deltas: 100% (53/53), done.\n",
            "Cloning into 'subword-nmt'...\n",
            "remote: Enumerating objects: 580, done.\u001b[K\n",
            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
            "remote: Total 580 (delta 0), reused 1 (delta 0), pack-reused 576\u001b[K\n",
            "Receiving objects: 100% (580/580), 237.41 KiB | 1.60 MiB/s, done.\n",
            "Resolving deltas: 100% (349/349), done.\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IeYW2BJhlJvx",
        "outputId": "817f72f4-bc1c-4af5-b22e-2375d34e25ea"
      },
      "source": [
        "# Install the necessary libraries\n",
        "!pip install sacremoses pandas mock sacrebleu tensorboardX pyarrow indic-nlp-library\n",
        "! pip install mosestokenizer subword-nmt\n",
        "# Install fairseq from source\n",
        "!git clone https://github.com/pytorch/fairseq.git\n",
        "%cd fairseq\n",
        "\n",
        "!pip install --editable ./\n",
        "\n",
        "%cd .."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 2.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
            "Collecting mock\n",
            "  Downloading mock-4.0.3-py3-none-any.whl (28 kB)\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.0.0-py3-none-any.whl (90 kB)\n",
            "\u001b[K     |████████████████████████████████| 90 kB 8.4 MB/s \n",
            "\u001b[?25hCollecting tensorboardX\n",
            "  Downloading tensorboardX-2.4-py2.py3-none-any.whl (124 kB)\n",
            "\u001b[K     |████████████████████████████████| 124 kB 45.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow in /usr/local/lib/python3.7/dist-packages (3.0.0)\n",
            "Collecting indic-nlp-library\n",
            "  Downloading indic_nlp_library-0.81-py3-none-any.whl (40 kB)\n",
            "\u001b[K     |████████████████████████████████| 40 kB 5.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses) (1.15.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sacremoses) (4.62.3)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacremoses) (2019.12.20)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses) (7.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: numpy>=1.15.4 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.19.5)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (0.8.9)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.3.2-py2.py3-none-any.whl (15 kB)\n",
            "Collecting colorama\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.17.3)\n",
            "Collecting morfessor\n",
            "  Downloading Morfessor-2.0.6-py3-none-any.whl (35 kB)\n",
            "Collecting sphinx-argparse\n",
            "  Downloading sphinx_argparse-0.3.1-py2.py3-none-any.whl (12 kB)\n",
            "Collecting sphinx-rtd-theme\n",
            "  Downloading sphinx_rtd_theme-1.0.0-py2.py3-none-any.whl (2.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8 MB 41.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: sphinx>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx-argparse->indic-nlp-library) (1.8.5)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.1.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.7.12)\n",
            "Requirement already satisfied: requests>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.23.0)\n",
            "Requirement already satisfied: Jinja2>=2.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.11.3)\n",
            "Requirement already satisfied: docutils>=0.11 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (0.17.1)\n",
            "Requirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.6.1)\n",
            "Requirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (21.0)\n",
            "Requirement already satisfied: sphinxcontrib-websupport in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.2.4)\n",
            "Requirement already satisfied: babel!=2.0,>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.9.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (57.4.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2>=2.3->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.0.0->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2021.5.30)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (2.4.7)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml in /usr/local/lib/python3.7/dist-packages (from sphinxcontrib-websupport->sphinx>=1.2.0->sphinx-argparse->indic-nlp-library) (1.1.5)\n",
            "Installing collected packages: sphinx-rtd-theme, sphinx-argparse, portalocker, morfessor, colorama, tensorboardX, sacremoses, sacrebleu, mock, indic-nlp-library\n",
            "Successfully installed colorama-0.4.4 indic-nlp-library-0.81 mock-4.0.3 morfessor-2.0.6 portalocker-2.3.2 sacrebleu-2.0.0 sacremoses-0.0.46 sphinx-argparse-0.3.1 sphinx-rtd-theme-1.0.0 tensorboardX-2.4\n",
            "Collecting mosestokenizer\n",
            "  Downloading mosestokenizer-1.1.0.tar.gz (37 kB)\n",
            "Collecting subword-nmt\n",
            "  Downloading subword_nmt-0.3.7-py2.py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: docopt in /usr/local/lib/python3.7/dist-packages (from mosestokenizer) (0.6.2)\n",
            "Collecting openfile\n",
            "  Downloading openfile-0.0.7-py3-none-any.whl (2.4 kB)\n",
            "Collecting uctools\n",
            "  Downloading uctools-1.3.0.tar.gz (4.6 kB)\n",
            "Collecting toolwrapper\n",
            "  Downloading toolwrapper-2.1.0.tar.gz (3.2 kB)\n",
            "Building wheels for collected packages: mosestokenizer, toolwrapper, uctools\n",
            "  Building wheel for mosestokenizer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mosestokenizer: filename=mosestokenizer-1.1.0-py3-none-any.whl size=49117 sha256=2164036ade47d1d00b80c42409989cb344c050043284c34151696831006f5d7c\n",
            "  Stored in directory: /root/.cache/pip/wheels/a7/31/94/fef279382208e85a65c1a7f5c4d0020115477b0af74f296b57\n",
            "  Building wheel for toolwrapper (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for toolwrapper: filename=toolwrapper-2.1.0-py3-none-any.whl size=3354 sha256=139195374012e4c3b07ad984a58f20e3a4e5c65fdd5af5dc32e18ed8da8a1f5b\n",
            "  Stored in directory: /root/.cache/pip/wheels/c5/4f/33/54741ffe08e38ececb1d28068a153729b4fe820bafa0a0691f\n",
            "  Building wheel for uctools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for uctools: filename=uctools-1.3.0-py3-none-any.whl size=6163 sha256=6b259f26e24ad557ce9ee523d6cb25333f27a6f81e1261dcc88a5b610c65202e\n",
            "  Stored in directory: /root/.cache/pip/wheels/fb/44/e9/914cf8fa71f0141f9314f862538d1218fcf2b94542a0fb7d35\n",
            "Successfully built mosestokenizer toolwrapper uctools\n",
            "Installing collected packages: uctools, toolwrapper, openfile, subword-nmt, mosestokenizer\n",
            "Successfully installed mosestokenizer-1.1.0 openfile-0.0.7 subword-nmt-0.3.7 toolwrapper-2.1.0 uctools-1.3.0\n",
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 29549, done.\u001b[K\n",
            "remote: Counting objects: 100% (630/630), done.\u001b[K\n",
            "remote: Compressing objects: 100% (463/463), done.\u001b[K\n",
            "remote: Total 29549 (delta 214), reused 470 (delta 155), pack-reused 28919\u001b[K\n",
            "Receiving objects: 100% (29549/29549), 13.57 MiB | 19.45 MiB/s, done.\n",
            "Resolving deltas: 100% (21943/21943), done.\n",
            "/content/fairseq\n",
            "Obtaining file:///content/fairseq\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (2019.12.20)\n",
            "Requirement already satisfied: sacrebleu>=1.4.12 in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (2.0.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (4.62.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (1.19.5)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (0.29.24)\n",
            "Collecting bitarray\n",
            "  Downloading bitarray-2.3.4.tar.gz (88 kB)\n",
            "\u001b[K     |████████████████████████████████| 88 kB 2.1 MB/s \n",
            "\u001b[?25hCollecting torchaudio>=0.8.0\n",
            "  Downloading torchaudio-0.9.1-cp37-cp37m-manylinux1_x86_64.whl (1.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9 MB 11.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (1.9.0+cu102)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from fairseq==1.0.0a0+666d8c2) (1.14.6)\n",
            "Collecting omegaconf<2.1\n",
            "  Downloading omegaconf-2.0.6-py3-none-any.whl (36 kB)\n",
            "Collecting hydra-core<1.1,>=1.0.7\n",
            "  Downloading hydra_core-1.0.7-py3-none-any.whl (123 kB)\n",
            "\u001b[K     |████████████████████████████████| 123 kB 46.5 MB/s \n",
            "\u001b[?25hCollecting antlr4-python3-runtime==4.8\n",
            "  Downloading antlr4-python3-runtime-4.8.tar.gz (112 kB)\n",
            "\u001b[K     |████████████████████████████████| 112 kB 36.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from hydra-core<1.1,>=1.0.7->fairseq==1.0.0a0+666d8c2) (5.2.2)\n",
            "Collecting PyYAML>=5.1.*\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 37.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from omegaconf<2.1->fairseq==1.0.0a0+666d8c2) (3.7.4.3)\n",
            "Requirement already satisfied: portalocker in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+666d8c2) (2.3.2)\n",
            "Requirement already satisfied: colorama in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+666d8c2) (0.4.4)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu>=1.4.12->fairseq==1.0.0a0+666d8c2) (0.8.9)\n",
            "Collecting torch\n",
            "  Downloading torch-1.9.1-cp37-cp37m-manylinux1_x86_64.whl (831.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 831.4 MB 6.7 kB/s \n",
            "\u001b[?25hRequirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->fairseq==1.0.0a0+666d8c2) (2.20)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.7/dist-packages (from importlib-resources->hydra-core<1.1,>=1.0.7->fairseq==1.0.0a0+666d8c2) (3.5.0)\n",
            "Building wheels for collected packages: antlr4-python3-runtime, bitarray\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-py3-none-any.whl size=141230 sha256=8542bac09fc1344ffb9011fd0639c8772b268ade79b5f233e2447a2969138919\n",
            "  Stored in directory: /root/.cache/pip/wheels/ca/33/b7/336836125fc9bb4ceaa4376d8abca10ca8bc84ddc824baea6c\n",
            "  Building wheel for bitarray (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for bitarray: filename=bitarray-2.3.4-cp37-cp37m-linux_x86_64.whl size=171973 sha256=6fac3e56777c0d4f90e647f5685f59b5deb2044e3042d26ec56d780443fe9af3\n",
            "  Stored in directory: /root/.cache/pip/wheels/84/cc/5b/0e861bdb5294d22d2d4f595df936f964a95258387e11494d41\n",
            "Successfully built antlr4-python3-runtime bitarray\n",
            "Installing collected packages: PyYAML, torch, omegaconf, antlr4-python3-runtime, torchaudio, hydra-core, bitarray, fairseq\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: torch\n",
            "    Found existing installation: torch 1.9.0+cu102\n",
            "    Uninstalling torch-1.9.0+cu102:\n",
            "      Successfully uninstalled torch-1.9.0+cu102\n",
            "  Running setup.py develop for fairseq\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "torchvision 0.10.0+cu102 requires torch==1.9.0, but you have torch 1.9.1 which is incompatible.\n",
            "torchtext 0.10.0 requires torch==1.9.0, but you have torch 1.9.1 which is incompatible.\u001b[0m\n",
            "Successfully installed PyYAML-5.4.1 antlr4-python3-runtime-4.8 bitarray-2.3.4 fairseq-1.0.0a0+666d8c2 hydra-core-1.0.7 omegaconf-2.0.6 torch-1.9.1 torchaudio-0.9.1\n",
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TktUu9NW_PLq"
      },
      "source": [
        "# this step is only required if you are running the code on colab\n",
        "# restart the runtime after running prev cell (to update). See this -> https://stackoverflow.com/questions/57838013/modulenotfounderror-after-successful-pip-install-in-google-colaboratory\n",
        "\n",
        "# this import will not work without restarting runtime\n",
        "from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E_4JxNdRlPQB",
        "outputId": "0b69b9bd-d38b-4b49-c327-b4f11538eb7f"
      },
      "source": [
        "# download the indictrans model\n",
        "\n",
        "\n",
        "# downloading the indic-en model\n",
        "!wget https://storage.googleapis.com/samanantar-public/V0.2/models/indic-en.zip\n",
        "!unzip indic-en.zip\n",
        "\n",
        "# downloading the en-indic model\n",
        "# !wget https://storage.googleapis.com/samanantar-public/V0.2/models/en-indic.zip\n",
        "# !unzip en-indic.zip\n",
        "\n",
        "# # downloading the indic-indic model\n",
        "# !wget https://storage.googleapis.com/samanantar-public/V0.3/models/m2m.zip\n",
        "# !unzip m2m.zip\n",
        "\n",
        "%cd indicTrans"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2021-10-01 10:53:16--  https://storage.googleapis.com/samanantar-public/V0.2/models/indic-en.zip\n",
            "Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.191.128, 173.194.74.128, 173.194.192.128, ...\n",
            "Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.191.128|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4551079075 (4.2G) [application/zip]\n",
            "Saving to: ‘indic-en.zip’\n",
            "\n",
            "indic-en.zip        100%[===================>]   4.24G  38.6MB/s    in 78s     \n",
            "\n",
            "2021-10-01 10:54:34 (55.7 MB/s) - ‘indic-en.zip’ saved [4551079075/4551079075]\n",
            "\n",
            "Archive:  indic-en.zip\n",
            "   creating: indic-en/\n",
            "   creating: indic-en/vocab/\n",
            "  inflating: indic-en/vocab/bpe_codes.32k.SRC  \n",
            "  inflating: indic-en/vocab/vocab.SRC  \n",
            "  inflating: indic-en/vocab/vocab.TGT  \n",
            "  inflating: indic-en/vocab/bpe_codes.32k.TGT  \n",
            "   creating: indic-en/final_bin/\n",
            "  inflating: indic-en/final_bin/dict.TGT.txt  \n",
            "  inflating: indic-en/final_bin/dict.SRC.txt  \n",
            "   creating: indic-en/model/\n",
            "  inflating: indic-en/model/checkpoint_best.pt  \n",
            "/content/indicTrans\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yTnWbHqY01-B",
        "outputId": "d9ec727d-c1cc-47c9-eebd-35c6648ea5a2"
      },
      "source": [
        "from indicTrans.inference.engine import Model\n",
        "indic2en_model = Model(expdir='../indic-en')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Initializing vocab and bpe\n",
            "Initializing model for translation\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2021-10-01 10:57:27 | INFO | fairseq.tasks.translation | [SRC] dictionary: 35864 types\n",
            "2021-10-01 10:57:27 | INFO | fairseq.tasks.translation | [TGT] dictionary: 32088 types\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VFXrCNZGEN7Z"
      },
      "source": [
        "ta_paragraph =\"\"\"झोपड़े के द्वार पर बाप और बेटा दोनों एक बुझे हुए अलाव के सामने चुपचाप बैठे हुए हैं और अंदर बेटे की जवान बीबी बुधिया प्रसव वेदना में पछाड़ खा रही थी. रह रहकर उसके मुंह से ऐसी दिल हिला देने वाली आवाज़ निकलती थी, कि दोनों कलेजा थाम लेते थे. जाड़ों की रात थी, प्रकृति सन्नाटे में डूबी हुई, सारा गांव अंधकार में लय हो गया था.चमारों का कुनबा था और सारे गांव में बदनाम. घीसू एक दिन काम करता तो तीन दिन आराम करता. माधव इतना काम चोर था कि आध घंटे काम करता तो घंटे भर चिलम पीता. इसलिए उन्हें कहीं मजदूरी नहीं मिलती थी. घर में मुठ्ठी भर भी अनाज मौजूद हो, तो उनके लिए काम करने की कसम थी. जब दो चार फाके हो जाते तो घीसू पेड़ पर चढक़र लकडिय़ां तोड़ लाता और माधव बाजार से बेच लाता और जब तक वह पैसे रहते, दोनों इधर उधर मारे मारे फिरते.\n",
        "\n",
        "गांव में काम की कमी न थी. किसानों का गांव था, मेहनती आदमी के लिए पचास काम थे. मगर इन दोनों को उसी वक्त बुलाते, जब दो आदमियों से एक का काम पाकर भी सन्तोष कर लेने के सिवा और कोई चारा न होता. अगर दोनो साधु होते, तो उन्हें सन्तोष और धैर्य के लिए, संयम और नियम की बिलकुल जरूरत न होती. यह तो इनकी प्रकृति थी. विचित्र जीवन था इनका! घर में मिट्टी के दो चार बर्तन के सिवा कोई सम्पत्ति नहीं. फटे चीथड़ों से अपनी नग्नता को ढांके हुए जिये जाते थे. संसार की चिन्ताओं से मुक्त कर्ज से लदे हुए. गालियां भी खाते, मार भी खाते, मगर कोई गम नहीं. दीन इतने कि वसूली की बिलकुल आशा न रहने पर भी लोग इन्हें कुछ न कुछ कर्ज दे देते थे. मटर, आलू की फसल में दूसरों के खेतों से मटर या आलू उखाड़ लाते और भून भानकर खा लेते या दस पांच ऊख उखाड़ लाते और रात को चूसते.\n",
        "\n",
        "घीसू ने इसी आकाश वृत्ति से साठ साल की उम्र काट दी और माधव भी सपूत बेटे की तरह बाप ही के पद चिह्नों पर चल रहा था, बल्कि उसका नाम और भी उजागर कर रहा था. इस वक्त भी दोनों अलाव के सामने बैठकर आलू भून रहे थे, जो कि किसी खेत से खोद लाये थे. घीसू की स्त्री का तो बहुत दिन हुए, देहान्त हो गया था. माधव का ब्याह पिछले साल हुआ था. जब से यह औरत आयी थी, उसने इस खानदान में व्यवस्था की नींव डाली थी और इन दोनों बे गैरतों का दोजख भरती रहती थी. जब से वह आयी, यह दोनों और भी आरामतलब हो गये थे. बल्कि कुछ अकडऩे भी लगे थे. कोई कार्य करने को बुलाता, तो निब्र्याज भाव से दुगुनी मजदूरी मांगते. वही औरत आज प्रसव वेदना से मर रही थी और यह दोनों इसी इन्तजार में थे कि वह मर जाए, तो आराम से सोयें.\n",
        "\n",
        "घीसू ने आलू निकालकर छीलते हुए कहा. जाकर देख तो, क्या दशा है उसकी? चुड़ैल का फिसाद होगा, और क्या? यहां तो ओझा भी एक रुपया मांगता है!\n",
        "\n",
        "माधव को भय था, कि वह कोठरी में गया, तो घीसू आलुओं का बड़ा भाग साफ कर देगा. बोला. मुझे वहां जाते डर लगता है.\n",
        "\n",
        "‘डर किस बात का है, मैं तो यहां हूं ही.’\n",
        "\n",
        "‘तो तुम्हीं जाकर देखो न?’\n",
        "\n",
        "‘मेरी औरत जब मरी थी, तो मैं तीन दिन तक उसके पास से हिला तक नहीं; और फिर मुझसे लजाएगी कि नहीं? जिसका कभी मुंह नहीं देखा, आज उसका उघड़ा हुआ बदन देखूं! उसे तन की सुध भी तो न होगी? मुझे देख लेगी तो खुलकर हाथ पांव भी न पटक सकेगी!’\n",
        "\n",
        "‘मैं सोचता हूं कोई बाल बच्चा हुआ, तो क्या होगा? सोंठ, गुड़, तेल, कुछ भी तो नहीं है घर में!’\n",
        "\n",
        "‘सब कुछ आ जाएगा. भगवान दें तो! जो लोग अभी एक पैसा नहीं दे रहे हैं, वे ही कल बुलाकर रुपये देंगे. मेरे नौ लड़के हुए, घर में कभी कुछ न था; मगर भगवान ने किसी न किसी तरह बेड़ा पार ही लगाया.’\n",
        "\n",
        "जिस समाज में रात दिन मेहनत करने वालों की हालत उनकी हालत से कुछ बहुत अच्छी न थी, और किसानों के मुकाबले में वे लोग, जो किसानों की दुर्बलताओं से लाभ उठाना जानते थे, कहीं ज्यादा सम्पन्न थे, वहां इस तरह की मनोवृत्ति का पैदा हो जाना कोई अचरज की बात न थी. हम तो कहेंगे, घीसू किसानों से कहीं ज्यादा विचारवान था और किसानों के विचार शून्य समूह में शामिल होने के बदले बैठकबाजों की कुत्सित मण्डली में जा मिला था. हां, उसमें यह शक्ति न थी, कि बैठकबाजों के नियम और नीति का पालन करता. इसलिए जहां उसकी मण्डली के और लोग गांव के सरगना और मुखिया बने हुए थे, उस पर सारा गांव उंगली उठाता था. फिर भी उसे यह तसकीन तो थी ही कि अगर वह फटेहाल है तो कम से कम उसे किसानों की सी जी तोड़ मेहनत तो नहीं करनी पड़ती, और उसकी सरलता और निरीहता से दूसरे लोग बेजा फायदा तो नहीं उठाते!\n",
        "\n",
        "दोनों आलू निकाल निकालकर जलते जलते खाने लगे. कल से कुछ नहीं खाया था. इतना सब्र न था कि ठण्डा हो जाने दें. कई बार दोनों की जबानें जल गयीं. छिल जाने पर आलू का बाहरी हिस्सा जबान, हलक और तालू को जला देता था और उस अंगारे को मुंह में रखने से ज्यादा खैरियत इसी में थी कि वह अंदर पहुंच जाए. वहां उसे ठण्डा करने के लिए काफी सामान थे. इसलिए दोनों जल्द जल्द निगल जाते. हालांकि इस कोशिश में उनकी आंखों से आंसू निकल आते.\n",
        "\n",
        "घीसू को उस वक्त ठाकुर की बरात याद आयी, जिसमें बीस साल पहले वह गया था. उस दावत में उसे जो तृप्ति मिली थी, वह उसके जीवन में एक याद रखने लायक बात थी, और आज भी उसकी याद ताजी थी, बोला वह भोज नहीं भूलता. तब से फिर उस तरह का खाना और भरपेट नहीं मिला. लडक़ी वालों ने सबको भर पेट पूडिय़ां खिलाई थीं, सबको! छोटे बड़े सबने पूडिय़ां खायीं और असली घी की! चटनी, रायता, तीन तरह के सूखे साग, एक रसेदार तरकारी, दही, चटनी, मिठाई, अब क्या बताऊं कि उस भोज में क्या स्वाद मिला, कोई रोक टोक नहीं थी, जो चीज चाहो, मांगो, जितना चाहो, खाओ. लोगों ने ऐसा खाया, ऐसा खाया, कि किसी से पानी न पिया गया. मगर परोसने वाले हैं कि पत्तल में गर्म गर्म, गोल गोल सुवासित कचौडिय़ां डाल देते हैं. मना करते हैं कि नहीं चाहिए, पत्तल पर हाथ रोके हुए हैं, मगर वह हैं कि दिये जाते हैं. और जब सबने मुंह धो लिया, तो पान इलायची भी मिली. मगर मुझे पान लेने की कहां सुध थी? खड़ा हुआ न जाता था. चटपट जाकर अपने कम्बल पर लेट गया. ऐसा दिल दरियाव था वह ठाकुर!\n",
        "\n",
        "माधव ने इन पदार्थों का मन ही मन मजा लेते हुए कहा अब हमें कोई ऐसा भोज नहीं खिलाता.\n",
        "\n",
        "‘अब कोई क्या खिलाएगा? वह जमाना दूसरा था. अब तो सबको किफायत सूझती है. सादी ब्याह में मत खर्च करो, क्रिया कर्म में मत खर्च करो. पूछो, गरीबों का माल बटोर बटोरकर कहां रखोगे? बटोरने में तो कमी नहीं है. हां, खर्च में किफायत सूझती है!’\n",
        "\n",
        "‘तुमने एक बीस पूरियां खायी होंगी?’\n",
        "\n",
        "‘बीस से ज्यादा खायी थीं!’\n",
        "\n",
        "‘मैं पचास खा जाता!’\n",
        "\n",
        "‘पचास से कम मैंने न खायी होंगी. अच्छा पका था. तू तो मेरा आधा भी नहीं है.’\n",
        "\n",
        "आलू खाकर दोनों ने पानी पिया और वहीं अलाव के सामने अपनी धोतियां ओढ़कर पांव पेट में डाले सो रहे. जैसे दो बड़े बड़े अजगर गेंडुलिया मारे पड़े हों.\n",
        "\n",
        "और बुधिया अभी तक कराह रही थी.\n",
        "\n",
        "सबेरे माधव ने कोठरी में जाकर देखा, तो उसकी स्त्री ठण्डी हो गयी थी. उसके मुंह पर मक्खियां भिनक रही थीं. पथराई हुई आंखें ऊपर टंगी हुई थीं. सारी देह धूल से लथपथ हो रही थी. उसके पेट में बच्चा मर गया था.\n",
        "\n",
        "माधव भागा हुआ घीसू के पास आया. फिर दोनों जोर जोर से हाय हाय करने और छाती पीटने लगे. पड़ोस वालों ने यह रोना धोना सुना, तो दौड़े हुए आये और पुरानी मर्यादा के अनुसार इन अभागों को समझाने लगे.\n",
        "\n",
        "मगर ज्यादा रोने पीटने का अवसर न था. कफ़न की और लकड़ी की फिक्र करनी थी. घर में तो पैसा इस तरह गायब था, जैसे चील के घोंसले में मांस?\n",
        "\n",
        "बाप बेटे रोते हुए गांव के जमींदार के पास गये. वह इन दोनों की सूरत से नफ़रत करते थे. कई बार इन्हें अपने हाथों से पीट चुके थे. चोरी करने के लिए, वादे पर काम पर न आने के लिए. पूछा क्या है बे घिसुआ, रोता क्यों है? अब तो तू कहीं दिखलाई भी नहीं देता! मालूम होता है, इस गांव में रहना नहीं चाहता.\n",
        "\n",
        "घीसू ने जमीन पर सिर रखकर आंखों में आंसू भरे हुए कहा सरकार! बड़ी विपत्ति में हूं. माधव की घरवाली रात को गुजर गयी. रात भर तड़पती रही सरकार! हम दोनों उसके सिरहाने बैठे रहे. दवा दारू जो कुछ हो सका, सब कुछ किया, मुदा वह हमें दगा दे गयी. अब कोई एक रोटी देने वाला भी न रहा मालिक! तबाह हो गये. घर उजड़ गया. आपका गुलाम हूं, अब आपके सिवा कौन उसकी मिट्टी पार लगाएगा. हमारे हाथ में तो जो कुछ था, वह सब तो दवा दारू में उठ गया. सरकार ही की दया होगी, तो उसकी मिट्टी उठेगी. आपके सिवा किसके द्वार पर जाऊं.\n",
        "\n",
        "जमींदार साहब दयालु थे. मगर घीसू पर दया करना काले कम्बल पर रंग चढ़ाना था. जी में तो आया, कह दें, चल, दूर हो यहां से. यों तो बुलाने से भी नहीं आता, आज जब गरज पड़ी तो आकर खुशामद कर रहा है. हरामखोर कहीं का, बदमाश! लेकिन यह क्रोध या दण्ड देने का अवसर न था. जी में कुढ़ते हुए दो रुपये निकालकर फेंक दिए. मगर सान्त्वना का एक शब्द भी मुंह से न निकला. उसकी तरफ ताका तक नहीं. जैसे सिर का बोझ उतारा हो.\n",
        "\n",
        "जब जमींदार साहब ने दो रुपये दिये, तो गांव के बनिये महाजनों को इनकार का साहस कैसे होता? घीसू जमींदार के नाम का ढिंढोरा भी पीटना जानता था. किसी ने दो आने दिये, किसी ने चारे आने. एक घंटे में घीसू के पास पांच रुपये की अच्छी रकम जमा हो गयी. कहीं से अनाज मिल गया, कहीं से लकड़ी. और दोपहर को घीसू और माधव बाज़ार से कफ़न लाने चले. इधर लोग बांस वांस काटने लगे.\n",
        "\n",
        "गांव की नर्मदिल स्त्रियां आ आकर लाश देखती थीं और उसकी बेकसी पर दो बूंद आंसू गिराकर चली जाती थीं.\n",
        "\n",
        "बाज़ार में पहुंचकर घीसू बोला लकड़ी तो उसे जलाने भर को मिल गयी है, क्यों माधव!\n",
        "\n",
        "माधव बोला हां, लकड़ी तो बहुत है, अब कफ़न चाहिए.\n",
        "\n",
        "‘तो चलो, कोई हलका सा कफ़न ले लें.’\n",
        "\n",
        "‘हां, और क्या! लाश उठते उठते रात हो जाएगी. रात को कफ़न कौन देखता है?’\n",
        "\n",
        "‘कैसा बुरा रिवाज है कि जिसे जीते जी तन ढांकने को चीथड़ा भी न मिले, उसे मरने पर नया कफ़न चाहिए.’\n",
        "\n",
        "‘कफ़न लाश के साथ जल ही तो जाता है.’\n",
        "\n",
        "‘और क्या रखा रहता है? यही पांच रुपये पहले मिलते, तो कुछ दवा दारू कर लेते.’\n",
        "\n",
        "दोनों एक दूसरे के मन की बात ताड़ रहे थे. बाजार में इधर उधर घूमते रहे. कभी इस बजाज की दूकान पर गये, कभी उसकी दूकान पर! तरह तरह के कपड़े, रेशमी और सूती देखे, मगर कुछ जंचा नहीं. यहां तक कि शाम हो गयी. तब दोनों न जाने किस दैवी प्रेरणा से एक मधुशाला के सामने जा पहुंचे. और जैसे किसी पूर्व निश्चित व्यवस्था से अंदर चले गये. वहां जरा देर तक दोनों असमंजस में खड़े रहे. फिर घीसू ने गद्दी के सामने जाकर कहा साहूजी, एक बोतल हमें भी देना.\n",
        "\n",
        "उसके बाद कुछ चिखौना आया, तली हुई मछली आयी और दोनों बरामदे में बैठकर शान्तिपूर्वक पीने लगे.\n",
        "\n",
        "कई कुज्जियां ताबड़तोड़ पीने के बाद दोनों सरूर में आ गये.\n",
        "\n",
        "घीसू बोला कफ़न लगाने से क्या मिलता? आखिर जल ही तो जाता. कुछ बहू के साथ तो न जाता.\n",
        "\n",
        "माधव आसमान की तरफ देखकर बोला, मानों देवताओं को अपनी निष्पापता का साक्षी बना रहा हो दुनिया का दस्तूर है, नहीं लोग बांभनों को हजारों रुपये क्यों दे देते हैं? कौन देखता है, परलोक में मिलता है या नहीं!\n",
        "\n",
        "‘बड़े आदमियों के पास धन है, फ़ूंके. हमारे पास फूंकने को क्या है?’\n",
        "\n",
        "‘लेकिन लोगों को जवाब क्या दोगे? लोग पूछेंगे नहीं, कफ़न कहां है?’\n",
        "\n",
        "घीसू हंसा अबे, कह देंगे कि रुपये कमर से खिसक गये. बहुत ढूंढ़ा, मिले नहीं. लोगों को विश्वास न आएगा, लेकिन फिर वही रुपये देंगे.\n",
        "\n",
        "माधव भी हंसा इस अनपेक्षित सौभाग्य पर. बोला बड़ी अच्छी थी बेचारी! मरी तो खूब खिला पिलाकर!\n",
        "\n",
        "आधी बोतल से ज्यादा उड़ गयी. घीसू ने दो सेर पूडिय़ां मंगाई. चटनी, अचार, कलेजियां. शराबखाने के सामने ही दूकान थी. माधव लपककर दो पत्तलों में सारे सामान ले आया. पूरा डेढ़ रुपया खर्च हो गया. सिर्फ थोड़े से पैसे बच रहे.\n",
        "\n",
        "दोनों इस वक्त इस शान में बैठे पूडिय़ां खा रहे थे जैसे जंगल में कोई शेर अपना शिकार उड़ा रहा हो. न जवाबदेही का खौफ था, न बदनामी की फ़िक्र. इन सब भावनाओं को उन्होंने बहुत पहले ही जीत लिया था.\n",
        "\n",
        "घीसू दार्शनिक भाव से बोला हमारी आत्मा प्रसन्न हो रही है तो क्या उसे पुन्न न होगा?\n",
        "\n",
        "माधव ने श्रद्धा से सिर झुकाकर तसदीक़ की जरूर से जरूर होगा. भगवान्, तुम अन्तर्यामी हो. उसे बैकुण्ठ ले जाना. हम दोनों हृदय से आशीर्वाद दे रहे हैं. आज जो भोजन मिला वह कभी उम्र भर न मिला था.\n",
        "\n",
        "एक क्षण के बाद माधव के मन में एक शंका जागी. बोला क्यों दादा, हम लोग भी एक न एक दिन वहां जाएंगे ही?\n",
        "\n",
        "घीसू ने इस भोले भाले सवाल का कुछ उत्तर न दिया. वह परलोक की बातें सोचकर इस आनन्द में बाधा न डालना चाहता था.\n",
        "\n",
        "‘जो वहां हम लोगों से पूछे कि तुमने हमें कफ़न क्यों नहीं दिया तो क्या कहोगे?’\n",
        "\n",
        "‘कहेंगे तुम्हारा सिर!’\n",
        "\n",
        "‘पूछेगी तो जरूर!’\n",
        "\n",
        "‘तू कैसे जानता है कि उसे कफ़न न मिलेगा? तू मुझे ऐसा गधा समझता है? साठ साल क्या दुनिया में घास खोदता रहा हूं? उसको कफ़न मिलेगा और बहुत अच्छा मिलेगा!’\n",
        "\n",
        "माधव को विश्वास न आया. बोला कौन देगा? रुपये तो तुमने चट कर दिये. वह तो मुझसे पूछेगी. उसकी मांग में तो सेंदुर मैंने डाला था.\n",
        "\n",
        "‘कौन देगा, बताते क्यों नहीं?’\n",
        "\n",
        "‘वही लोग देंगे, जिन्होंने अबकी दिया. हां, अबकी रुपये हमारे हाथ न आएंगे.’\n",
        "\n",
        "‘ज्यों ज्यों अंधेरा बढ़ता था और सितारों की चमक तेज होती थी, मधुशाला की रौनक भी बढ़ती जाती थी. कोई गाता था, कोई डींग मारता था, कोई अपने संगी के गले लिपटा जाता था. कोई अपने दोस्त के मुंह में कुल्हड़ लगाये देता था.’\n",
        "\n",
        "वहां के वातावरण में सरूर था, हवा में नशा. कितने तो यहां आकर एक चुल्लू में मस्त हो जाते थे. शराब से ज्यादा यहां की हवा उन पर नशा करती थी. जीवन की बाधाएं यहां खींच लाती थीं और कुछ देर के लिए यह भूल जाते थे कि वे जीते हैं या मरते हैं. या न जीते हैं, न मरते हैं.\n",
        "\n",
        "और यह दोनों बाप बेटे अब भी मजे ले लेकर चुसकियां ले रहे थे. सबकी निगाहें इनकी ओर जमी हुई थीं. दोनों कितने भाग्य के बली हैं! पूरी बोतल बीच में है.\n",
        "\n",
        "भरपेट खाकर माधव ने बची हुई पूडिय़ों का पत्तल उठाकर एक भिखारी को दे दिया, जो खड़ा इनकी ओर भूखी आंखों से देख रहा था. और देने के गौरव, आनन्द और उल्लास का अपने जीवन में पहली बार अनुभव किया.\n",
        "\n",
        "घीसू ने कहा ले जा, खूब खा और आशीर्वाद दे! जिसकी कमाई है, वह तो मर गयी. मगर तेरा आशीर्वाद उसे जरूर पहुंचेगा. रोयें रोयें से आशीर्वाद दो, बड़ी गाढ़ी कमाई के पैसे हैं!\n",
        "\n",
        "माधव ने फिर आसमान की तरफ देखकर कहा वह बैकुण्ठ में जाएगी दादा, बैकुण्ठ की रानी बनेगी.\n",
        "\n",
        "घीसू खड़ा हो गया और जैसे उल्लास की लहरों में तैरता हुआ बोला हां, बेटा बैकुण्ठ में जाएगी. किसी को सताया नहीं, किसी को दबाया नहीं. मरते मरते हमारी जिन्दगी की सबसे बड़ी लालसा पूरी कर गयी. वह न बैकुण्ठ जाएगी तो क्या ये मोटे मोटे लोग जाएंगे, जो गरीबों को दोनों हाथों से लूटते हैं, और अपने पाप को धोने के लिए गंगा में नहाते हैं और मन्दिरों में जल चढ़ाते हैं?\n",
        "\n",
        "श्रद्धालुता का यह रंग तुरन्त ही बदल गया. अस्थिरता नशे की खासियत है. दु:ख और निराशा का दौरा हुआ.\n",
        "\n",
        "माधव बोला मगर दादा, बेचारी ने जिन्दगी में बड़ा दु:ख भोगा. कितना दु:ख झेलकर मरी!\n",
        "\n",
        "वह आंखों पर हाथ रखकर रोने लगा. चीखें मार मारकर.\n",
        "\n",
        "घीसू ने समझाया क्यों रोता है बेटा, खुश हो कि वह माया जाल से मुक्त हो गयी, जंजाल से छूट गयी. बड़ी भाग्यवान थी, जो इतनी जल्द माया मोह के बन्धन तोड़ दिये.\n",
        "\n",
        "और दोनों खड़े होकर गाने लगे \"\"\"\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hi_D7s_VIjis",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "52f34eff-dea0-4531-e32a-0ba9f63f86bb"
      },
      "source": [
        "story = indic2en_model.translate_paragraph(ta_paragraph, 'hi', 'en')\n",
        "\n",
        "story"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 244/244 [00:00<00:00, 10234.20it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  ../aten/src/ATen/native/BinaryOps.cpp:467.)\n",
            "  return torch.floor_divide(self, other)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'At the door of the hut, both the father and the son were sitting quietly in front of a buried bonfire, and inside, the son \\'s soldier, Bibi Budhia, was writhing in labour. There was such a heart - wrenching sound coming out of his mouth that both of them held the collage together. It was a winter night, nature was in a state of silence, the whole village was in darkness. There was a family of Chamars and the whole village was infamous. If Ghisu worked for one day, he would rest for three days. Madhav was so busy that if he worked for half an hour, he would drink for an hour. So they didnt take it. If there was even a handful of grain in the house, I swore to work for them. When it was two or four fissures away, Ghisu would climb up the tree and break the wood and Madhav would sell it from the market and as long as the money was there, both would roam around. There was no employment in the village. There was a village of farmers, there were fifty jobs for the hard working man. But he would call them both when they had no choice but to be satisfied with a single job from two men. If both of them were sadhus, they would not need any restraint and discipline for satisfaction and patience. That was his nature. It was a strange life! There is no property in the house except two or four earthen pots. They lived with torn rags covering their nakedness. loaded with debt free from the concerns of the world. They were also abused, beaten, but no one was hurt. So much so that even when there was no hope of recovery, people lent them something or the other. The pea, in the potato crop, would pluck the pea or potato from the fields of others and eat roasted or would pick up ten or five husks and suck at night. Ghisu finished sixty years of age with the same sky spirit, and Madhav, like his son, was following in his father \\'s footsteps, revealing his name even more. Even at this time, both of them were sitting in front of the bonfire, roasting potatoes, which had been dug out of a field. Ghisu \\'s wife had died a long time ago. Madhav was married last year. From the time this woman came, she had laid the foundation of the order in this family and these two unscrupulous men were in the fire of hell. When she arrived, they were both more comfortable. But there were some glitches too. If someone called for a job, he would ask for a double wage. The same woman was dying of labour today, and they were both waiting for her to die, sleeping comfortably. Ghisu took out the potatoes and peeled them. Go and see what he is like. The witch will fade, and what else? Here the oza is also asking for a rupee! Madhav was afraid that if he entered the room, Ghisu would clean up a large part of the potatoes. \"\" \"What am I afraid of?\" Do you want to go and see it? \\'When my wife was dead, I didn \\'t move from her for three days and then will I be ashamed of myself? Let me see the body of a man whose face I had never seen before. Does he not have a sense of well - being? If you see me, you won \\'t be able to move your hands and feet. What will happen if a child is born? Theres nothing at home, nothing at all! Everything will come. God bless it! Only those who are not giving a single penny will call and give money tomorrow. I had nine boys, there was never anything in the house, but God somehow made it across the fence. In a society where the condition of those who toiled day and night was no better than that of the peasants, and those who knew how to take advantage of the peasant \\'s weaknesses were far richer, it was no wonder that such an attitude was born. We would say that Ghisu was more thoughtful than the peasants, and that the ideas of the peasants, instead of being in a nondescript group, had got into the ugly assembly of the sitting-men. Yes, it did not have the power to follow the rules and policy of the sittings. Therefore, while other members of his congregation were the village head and head, the whole village pointed fingers at him. Yet he was convinced that if he was broken, he would not have to work as hard as the peasants, and that his simplicity and indifference would not be of any benefit to others. Both of them took out the potatoes and started eating them. He hasn\\'t eaten anything since yesterday. I didnt have the patience to let it cool down. Both had been stabbed several times. When the potato was peeled, the outer portion of the potato burned the tongue, the halk and the palate, and it was better to put the fire in the mouth than to put it in the mouth. There were plenty of things to cool him down. Therefore, the two were separated soon. However, tears welled up in his eyes in this effort. Ghisu remembered Tagore \\'s quote to which he had gone twenty years ago. The satisfaction he got at the feast was a memorable thing in his life, and it was still fresh in his mind, and he said that he never forgets the feast. There has been no food or water since then. The girls fed everyone a full meal, everyone! All young and old ate pudis and made real oil! Chutney, raita, three types of dried saag, a juicy vegetable, yogurt, chutney, sweets, what the taste of that meal was, there was no bar, anything you want, ask, eat as much as you want. People ate like this, ate like this, they didn\\'t drink like that. But they \\'re going to serve the leaf with hot, round, aromatic cashews. Refrain or not, hands are tied on the leaf, but they are given. And when everyone washed their mouths, they got a betel leaf. But where was I to drink? He did not stand. She immediately went and lay down on her sofa. He was such a brave heart! Madhav made fun of the food items and said, No one now serves us a feast like this. What will anyone eat then? That was the second time. Now everyone thinks it\\'s affordable. Do not spend in a simple marriage, do not spend in action. Ask, where will you collect the poor \\'s goods and keep them? There is no shortage of collection. Yes, it is cost - effective! \\'You must have eaten a twenty pudis? I ate more than twenty! I would have eaten fifty! I have not eaten less than fifty. It was cooked well. You are not even half of me. After eating the potatoes, the two of them drank water and slept in front of the bonfire, covered their dhows and put their feet in the stomach. like two big python rhinoceros killed. And Budhia was still crying. Madhav went to the room in the morning and found that his wife was cold. His mouth was full of flies. His eyes were closed. The entire body was covered with dust. The baby died in the womb. Madhav came to the runaway Ghisu. The two then started shouting and beating each other up. Neighbours heard the cry and rushed to the spot and tried to explain the misfortunes according to the old norms. But there was no time to cry. We had to worry about the coffins and the wood. Money was missing in the house like meat in the nest of a eagle? The father went weeping to the zamindar of the village. He hated both of them. They had assaulted him several times. To steal, not to come to work on the promise. Why are you crying? he asked. Now you are nowhere to be seen! I dont want to live in this village. Ghisu placed his head on the ground and said with tears in his eyes. Im in very big trouble. Madhav \\'s wife passed away last night. The government was silent all night long. We were seated next to him. Drug wine did everything it could, but it betrayed us. Now there was not a single breadwinner left. were destroyed. The house was gutted. I am your slave, who will cross his soil now except you. Everything we had in our hands got into the drug. If the government has mercy, the soil will rise. Who else can I go to? The zamindar was kind. But to have mercy on Ghisu was to paint the black blanket. He came to me, tell me, go away from here. It does not even come from calling, but today when it rains, it comes and rejoices. You bastards, you bastards! But this was not the time for anger or punishment. He took out Rs two lakh from the bag. But not a word of comfort came out of his mouth. He didnt even look at it. like a load off the head. When the zamindar gave two rupees, how dare the village mahajans refuse? Ghisu also knew how to beat up the name of the zamindar. Some gave me two, some gave me four. In one hour, Ghisu had a good sum of Rs. 5. Somewhere we got grain, somewhere we got wood. And in the afternoon Ghisu and Madhav went to fetch the kafan from the market. People started gathering here. The gentle women of the village used to come and see the dead bodies and go away with two drops of tears on her bare hands. When Ghisu reached the market, he said, The wood has got enough to burn it, why Madhav! Madhav said, Yes, there is enough wood, now we need kafan. \"\" \"So let\\'s take a little bit of coffee.\" Yes, and what else! The dead body will rise in the evening. Who sees the night? It is a bad custom that a person who does not even get a scratch to cover his body while he is alive should have a new coffin when he dies. \"\" \"\" \"Kafan\" \"\" \"goes with the body only in water.\" What else remains? Had he got the same five rupees earlier, he would have got some medicine. \"\" \"Both of them were talking to each other.\" The market keeps moving up and down. Sometimes we went to Bajaj \\'s shop, sometimes to his shop. I saw all kinds of clothes, silk and cotton, but nothing matted. Even evening. Then, under divine inspiration, the two of them went to a madrasa. and as if they went in with a predetermined arrangement. The two remained together for sometime. Then Ghisu went to the throne and said, Sir, give us a bottle too. Then there came some chikhuna, fried fish and they both sat in the verandah and drank in peace. After drinking a lot of Kujis, both of them came to Sarur. What does it mean to wear a kafan? asked Ghisu. Finally, the water falls. Some did not go with their wives. Madhav looked up to the sky and said, It is the practice of the world to make gods witness to their innocence, but why do people give thousands of rupees to the Bambarans? Who will see whether in the future or not? \"\" \"Big people have money.\" What do we have to do? But what will people say? People will not ask, where is Kafan? \\'Gheesu Hasa Abe will say that the rupee has slipped from the waist. They searched for him quite a long but he didnt found. People will not believe it, but they will pay the same amount. Madhav also laughed at this unexpected fortune. He was so poor! It \\'s a lot to eat. More than half of the bottle flew. Ghisu ordered two sher pudis. Chutneys, pickles, colleges. There was a shop right in front of the shop. Madhav leapt in two leaves and carried all his belongings. A total of Rs. Only a little money is left. Both of them were sitting in the pride, eating the pudis, as if a lion were flying its prey in the forest. There was no fear of shame or guilt. He had won all these emotions long ago. Ghisu said in a philosophical voice, If our soul is happy, will it not be satisfied? Madhav bowed his head reverently and confirmed that he must. God, you\\'re inanimate. Take him to the backseat. We are both blessing each other wholeheartedly. The food we got today was never available for life. A moment later, a doubt arose in Madhav \\'s mind. Why would we go there someday, Grandpa? Ghisu did not answer this nave question. He did not want to hinder this joy by thinking of the future. \"\" \"What would you say if someone asked us why we were not buried there?\" They will say \\'your head\\'! Ask a question! How do you know he will not get the kafan? Do you think Im nuts? Have I been digging the grass in the world for 60 years? He\\'ll get the kafan and he\\'ll get the best! \"\" \"I did not believe it.\" Who said what You have saved Rs. He will ask me. I put the centur on his demand. Why doesnt anyone say anything? It will be given by the people who have given it. Yes, the money is not in our hands. As darkness grew and the stars shone brightly, so did the beehive. Some sang, some sang, some danced, some hugged their mates. Someone would put an axe in his friend\\'s mouth. \"\" \"There was an air of smoke in the atmosphere.\" Some of them used to come here and have fun in a furnace. They were more intoxicated by the air here than by alcohol. The barriers of life dragged them here and for a while they forgot whether they lived or died. They dont live or die. And the father and son were still having fun and chuckling. Everyones eyes had settled on them. How fortunate they are both! The whole bottle is in the middle. Having eaten enough, Madhav picked up the leaves of the leftovers and gave them to a beggar who was standing there looking at them with hungry eyes. And for the first time in my life, I experienced the glory of giving, the joy and the ecstasy of giving. Gheesu said, Take it, eat a lot and bless! Those who earned, died. But your blessings will surely reach him. Bless me with roe roe, it \\'s a lot of hard money! Madhav then looked up to the sky and said that she would go to Bakunth and become the queen of Baikunth. Ghisu stood up and, as if floating in the waves of joy, said, \\'Yes, the son will go to the backseat. No one was hurt and no damage occurred. Death fulfilled the greatest desire of our lives. If she does not go, will these fat people go, who loot the poor with both hands, who bathe in the Ganges to wash their sins and offer water in the temples? This colour of devotion immediately changed. Instability is a characteristic of drugs. There was frustration and disappointment. Madhav said, But Grandpa, the poor girl had suffered a lot in her life. How tragic! He put his hands over his eyes and started crying. Scream and shout. Ghisu explained why the boy wept, happy that she was free from the maya net, free from the maze. I was lucky enough to break the shackles of Maya love so quickly. And both of them stood up and started singing'"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jFrPXpfRtoGC"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99TEmYY9DS-F"
      },
      "source": [
        "import nltk"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eTnjlzU2p1Rj"
      },
      "source": [
        "refrence = \"\"\" At the door of the hut father and son sat silently by a burnt-out fire; inside, the son's young wife Budhiya lay in labor, writhing with pain. And from time to time such a heart-rending scream emerged from her lips that they both pressed their hands to their hearts. It was a winter night; everything was drowned in desolation. The whole village had been absorbed into the darkness.\n",
        "\n",
        "Ghisu said, \"It seems she won't live. She's been writhing in pain the whole day. Go on-- see how she is.\"\n",
        "\n",
        "Madhav said in a pained tone, \"If she's going to die, then why doesn't she go ahead and die? What's the use of going to see?\"\n",
        "\n",
        "\"You’re pretty hard-hearted! You’ve enjoyed life with her for a whole year-- such faithlessness to her?\"\n",
        "\n",
        "\"Well, I can't stand to see her writhing and thrashing around.\"\n",
        "\n",
        "It was a family of Chamars, and notorious in the whole village. If Ghisu worked for one day, then he rested for three. Madhav was such a slacker that if he worked for an hour, then he smoked his chilam for an hour. Thus nobody hired them on. If there was even a handful of grain in the house, they both swore off working. When they'd fasted for a couple of days, then Ghisu climbed trees and broke off branches, and Madhav sold the wood in the market; and as long as that money lasted, they both spent their time wandering idly around. *When their hunger grew intense, they again broke off branches, or looked for some work.* There was no shortage of work in the village. It was a village of farmers; for a hard-working man there were fifty jobs. But people only sent for those two when they were forced to content themselves with getting out of two men the work of one.\n",
        "\n",
        "If only the two had been ascetics, then they wouldn't have needed any exercises in self-discipline to achieve contentment and patience. This was their very nature. Theirs was a strange life. Except for two or three clay pots, they had no goods at all in the house. Covering their nakedness with torn rags, free from the cares of the world, laden with debt-- they suffered abuse, they suffered blows too, but not grief. They were so poor that without the smallest hope of repayment, people used to lend them something or other. When peas or potatoes were in season, they would dig up peas or potatoes from the fields and roast and eat them, or break off five or ten stalks of sugarcane and suck them at night. Ghisu had spent sixty years of his life in this pious manner, and Madhav, like a dutiful son, was following in his father's footsteps-- or rather, was making his name even more radiant.\n",
        "\n",
        "*(1b)* This time too, both were seated by the fire, roasting potatoes that they had dug up from somebody's field. Ghisu's wife had passed away long ago. Madhav's marriage had taken place the year before. Since this woman had come, she had laid the foundations of civilization in the family. *Grinding grain, cutting grass, she arranged for a couple of pounds of flour,* and kept filling the stomachs of those two shameless ones. After she came, they both grew even more lazy and indolent; indeed, they even began to swagger a bit. If someone sent for them to work, then with splendid indifference they demanded double wages. That woman was dying today in childbirth. And these two were perhaps waiting for her to die, so they could sleep in peace.\n",
        "\n",
        "Pulling out a potato and peeling it, Ghisu said, \"Go see what shape she's in. We'll have the fuss over a ghost-witch-- what else! And here even the exorcist demands a rupee-- *from whose house would we get one?*\"\n",
        "\n",
        "Madhav suspected that if he went into the hut, Ghisu would finish off most of the potatoes. He said, \"I'm afraid to go in.\"\n",
        "\n",
        "\"What are you afraid of? I'm here, after all.\"\n",
        "\n",
        "\"Then you go and see, all right?\"\n",
        "\n",
        "\"When my wife died, for three days I never even left her side. And then, won't she be ashamed in front of me? I've never seen her face-- and today I should see her naked body? She won't even have bodily ease: if she sees me, she won't be able to thrash around freely.\"\n",
        "\n",
        "\"I'm thinking, if a child is born-- what then? Dried ginger, brown sugar, oil-- there's nothing at all in the house.\"\n",
        "\n",
        "\"Everything will come. If Bhagwan gives a child-- those people who now aren't giving a paisa, will send for us and give us things. I've had nine sons. There was never anything in the house, but this is how we managed every time.\"\n",
        "\n",
        "*(1c)* A society in which those who labored night and day were not in much better shape than these two; a society in which compared to the peasants, those who knew how to exploit the peasants' weaknesses were much better off-- in such a society, the birth of this kind of mentality was no cause for surprise. We'll say that compared to the peasants, Ghisu was more insightful; and instead of joining the mindless group of peasants, he had joined the group of clever, scheming tricksters. Though indeed, he wasn't skilful in following the rules and customs of the tricksters. Thus while other members of his group became chiefs and headmen of villages, at him the whole village wagged its finger. But still, he did have the consolation that if he was in bad shape, at least he wasn't forced to do the back-breaking labor of the peasants, and others didn't take improper advantage of his simplicity and voicelessness.\n",
        "\n",
        "Pulling out the potatoes, they both began to eat them burning hot. They had eaten nothing since the day before. They were too impatient to wait till the potatoes cooled. Both burned their tongues repeatedly. When the potatoes were peeled, their outer parts didn't seem so extremely hot. But the moment the teeth bit into them, the inner part burned the tongue and throat and roof of the mouth. Rather than keep that ember in the mouth, it was better to send it quickly along inward, where there was plenty of equipment for cooling it down. So they both swallowed very fast, although the attempt brought tears to their eyes.\n",
        "\n",
        "*(1d)* Then Ghisu remembered a landowner's wedding procession, in which he had taken part twenty years before. The repletion that had been vouchsafed to him in that feast was a memorable event in his life, and even today its memory was fresh. He said, \"I'll never forget that feast. Never since then have I had that kind of food, or such a full stomach. The girl's family fed puris to everyone. As much as they wanted! Great and small, everyone ate puris-- ones made with real ghi! Chutney, raita, three kinds of green vegetables, a flavorful stew, yoghurt, chutney, sweets. How can I tell you now what relish there was in that feast! There was no limit. Whatever thing you want, just ask! And however much you want, eat! People ate so much, ate so much, that nobody could even drink any water. And there the servers were-- setting hot, round, sweet-smelling pastries before you! You refuse, saying you don't want it. You push away the tray with your hand. But that's how they are-- they just keep on giving it. And when everybody had wiped their mouths, then everybody got a pan as well. But how could I be in any shape for a pan? I couldn't stand up. I just staggered off and lay down on my blanket. He had a heart as big as the ocean, that landowner!\"\n",
        "\n",
        "Enjoying the story of these grand festivities, Madhav said, \"If only somebody would give us such a feast now!\"\n",
        "\n",
        "\"As if anybody would feast anybody now! That was a different time. Now everybody thinks about economy-- 'don't spend money on weddings, don't spend money on religious festivals!'. Ask them-- what's this 'saving' of the poor people's wealth? There's no lack of 'saving'. But when it comes to spending, they think about economy!\"\n",
        "\n",
        "\"You must have eaten twenty or so puris?\"\n",
        "\n",
        "\"I ate more than twenty.\"\n",
        "\n",
        "\"I would have eaten up fifty.\"\n",
        "\n",
        "\"I couldn't have eaten less than fifty. I was hale and hearty. You're not half of what I was!\"\n",
        "\n",
        "After eating, they both drank some water, covered themselves with their dhotis, curled up, and went to sleep right there by the fire, as if two gigantic serpents lay coiled there.\n",
        "\n",
        "And Budhiya was still moaning.\n",
        "\n",
        "(2)\n",
        "\n",
        "*(2a)* In the morning, when Madhav went into the hut and looked, his wife had grown cold. Flies were buzzing on her face. Her stony eyes had rolled upward. Her whole body was covered with dust. In her stomach, the baby had died.\n",
        "\n",
        "Madhav came running to Ghisu. Then they both together began loudly lamenting and beating their breasts. When the neighbors heard the weeping and wailing, they came running. And following the ancient custom, they began to console the bereaved.\n",
        "\n",
        "But this wasn't the occasion for an excessive show of grief. They had to worry about the shroud, and the wood. Money was as scarce in their house as meat in a raptor's nest.\n",
        "\n",
        "Father and son went weeping to the village landlord. He hated the very sight of their faces. A number of times he had beaten them with his own hands-- for theft, or for not coming to work as they had promised. He asked, \"What is it, Ghisua, why do you weep? Nowadays we don't even see you around. It seems that you no longer want to live in the village.\"\n",
        "\n",
        "Ghisua fell prostrate on the ground, and said with tear-filled eyes, \"Master, I'm in great trouble! Madhav's wife passed away last night. All day she was writhing in pain, Master; we two sat by her bed till midnight. Whatever medicines we could give her, we did. But she slipped away. Now we have no one to care for us, Master-- we're devastated-- our house is destroyed! I'm your slave. Now who but you will take care of her final rites? Whatever money we had at hand was used up on medicines. If the Master will show mercy, then she'll have the proper rites. To whose door should I come except yours?\"\n",
        "\n",
        "*(2b)* The Landlord Sahib was a compassionate man. But to show compassion to Ghisu was to try to dye a black blanket. He felt like saying, \"Get out of here! *Keep the corpse in your house and let it rot!* Usually you don't come even when you're called-- now when you want something, you come and flatter me! You treacherous bastard! You villain!\" But this was not the occasion for anger or revenge. Willingly or not, he pulled out two rupees and flung them down. But he didn't open his lips to say a single word of consolation. He didn't even look in Ghisu's direction-- as if he'd discharged a duty.\n",
        "\n",
        "When the Landlord Sahib gave two rupees, then how could the village merchants and money-lenders have the nerve to refuse? Ghisu knew how to beat the drum of the landlord's name. One gave two paisas, another gave four paisas. In an hour, Ghisu had collected the sum of five rupees in ready cash. Someone gave grain, someone else gave wood. And in the afternoon Ghisu and Madhav went to the market to get a shroud. Meanwhile, people began to cut the bamboo poles, and so on.\n",
        "\n",
        "The sensitive-hearted women of the village came and looked at the body. They shed a few tears at its helplessness, and went away.\n",
        "\n",
        "(3)\n",
        "\n",
        "*(3a)* When they reached the market, Ghisu said, \"We've got enough wood to burn her, haven't we, Madhav?\"\n",
        "\n",
        "Madhav said, \"Yes, there's plenty of wood. Now we need a shroud.\"\n",
        "\n",
        "\"So let's buy a light kind of shroud.\"\n",
        "\n",
        "\"Sure, what else! While the body is being carried along, night will come. At night, who sees a shroud?\"\n",
        "\n",
        "\"What a bad custom it is that someone who didn't even get a rag to cover her body when she was alive, needs a new shroud when she's dead.\"\n",
        "\n",
        "\"After all, the shroud burns along with the body.\"\n",
        "\n",
        "\"What else is it good for? If we'd had these five rupees earlier, we would have given her some medicine.\"\n",
        "\n",
        "Each of them inwardly guessed what the other was thinking. They kept wandering here and there in the market, until eventually evening came. [Sometimes they went to one cloth-seller's shop, sometimes to another. They looked at various kinds of fabric, they looked at silk and cotton, but nothing suited them.] The two arrived, by chance or deliberately, before a wine-house; and as if according to some prearranged decision, they went inside. For a little while they both stood there in a state of uncertainty. [Then Ghisu went to the counter and said, \"Sir, please give us a bottle too.\"] *Ghisu bought one bottle of liquor, and some sesame sweets.* [After this some snacks came, fried fish came]. And they both sat down on the verandah and [peacefully] began to drink.\n",
        "\n",
        "*(3b)* After drinking a number of cups in a row, both became elevated.\n",
        "\n",
        "Ghisu said, \"What's the use of wrapping her in a shroud? After all, it would only be burned. Nothing would go with her.\"\n",
        "\n",
        "Looking toward the sky as if persuading the angels of his innocence, Madhav said, \"It's the custom of the world-- why do these same people give thousands of rupees to the Brahmins? Who can tell whether a reward does or doesn't reach them in another world?\"\n",
        "\n",
        "\"Rich people have wealth-- let them waste it! What do we have to waste?\"\n",
        "\n",
        "\"But what will you tell people? Won't people ask where the shroud is?\"\n",
        "\n",
        "Ghisu laughed. \"We'll say the money slipped out of my waistband-- we searched and searched for it, but it didn't turn up. [People won't believe it, but they'll still give the same sum again.]\"\n",
        "\n",
        "Madhav too laughed at this unexpected good fortune, *at defeating destiny in this way*. He said, \"She was very good, the poor thing. Even as she died, she gave us a fine meal.\"\n",
        "\n",
        "More than half the bottle had been finished. Ghisu ordered two sers of puris, a meat stew, and spiced liver and fried fish. There was a shop right next to the wine-house. Madhav ran over and brought everything back on two leaf-plates. The cost was fully one and a half rupees. Only a few paise were left.\n",
        "\n",
        "*(3c)* Both then sat eating puris, with all the majesty of a tiger in the jungle pursuing his prey. They had no fear of being called to account, nor any concern about disgrace. They had passed through these stages of weakness long ago. Ghisu said in a philosophical manner, \"If my soul is being pleased, then won't she receive religious merit?\"\n",
        "\n",
        "Madhav bowed his head in pious confirmation. \"Certainly she'll certainly receive it. Bhagwan, you are the knower of hearts-- take her to Heaven! We're both giving her our heartfelt blessing. The feast I've had today-- I haven't had its equal in my whole life!\"\n",
        "\n",
        "After a moment a doubt arose in Madhav's heart. He said, \"How about it-- we'll go there too someday, won't we?\"\n",
        "\n",
        "Ghisu gave no answer to this childish question. *He looked reproachfully at Madhav.* [He didn't want the thought of heavenly matters to interfere with this bliss.]\n",
        "\n",
        "\"When she asks us, there, why we didn't give her a shroud, what will you say?\"\n",
        "\n",
        "\"Oh, shut up!\"\n",
        "\n",
        "\"She'll certainly ask.\"\n",
        "\n",
        "\"How do you know that she won't get a shroud? Do you consider me such a donkey? I've lived in this world for sixty years-- and have I just been loitering around? She'll get a shroud, and [a very good one]-- *a much better than we would have given*.\"\n",
        "\n",
        "Madhav was not convinced. He said, \"Who will give it? You've gobbled up the rupees! [It's me she'll ask-- I'm the one who put the sindur in the parting of her hair.]\"\n",
        "\n",
        "Ghisu grew irritated. \"I tell you, she'll get a shroud. Why don't you believe me?\"\n",
        "\n",
        "\"Who will give the money-- why don't you tell me?\"\n",
        "\n",
        "\"The same people will give it who gave it this time. But they won't put the rupees into our hands. *And if somehow we get our hands on them, we'll sit here and drink again just like this, and they'll give the shroud a third time.*\"\n",
        "\n",
        "*(3d)* As the darkness deepened and and the stars glittered more sharply, the tumult in the wine-house also increased. One person sang, another babbled, another embraced his companion, another pressed a glass to his friend's lips. Joy was in the atmosphere there. Intoxication was in the air. How many people become 'an ass with a glass'! *They came here only to taste the pleasure of self-forgetfulness.* More than liquor, the air here elevated their spirits. The disaster of life seized them and dragged them here. And for a while they forgot whether they were alive or dead-- or half-alive.\n",
        "\n",
        "And these two, father and son, were still sipping with relish. Everyone's eyes had settled on them. How fortunate they were! They had a whole bottle between them.\n",
        "\n",
        "After he had finished eating, Madhav picked up the leaf-plate of leftover puris and gave it to a beggar who was standing there looking at them with hungry eyes. And for the first time in his life he felt the pride and delight and thrill of giving.\n",
        "\n",
        "Ghisu said, \"Take it-- eat your fill, and give her your blessing. She whose earnings these are has died, but your blessing will certainly reach her. Bless her with every hair on your body-- these are the payment for very hard labor.\"\n",
        "\n",
        "Madhav again looked toward the sky and said, \"She'll go to Heaven-- she'll become the Queen of Heaven!\"\n",
        "\n",
        "*(3e)* Ghisu stood up, and as if swimming in waves of joy he said, \"Yes, son, she'll go to Heaven! She never tormented anyone, she never oppressed anyone; even while dying, she fulfilled the greatest desire of our lives. If she doesn't go to Heaven, then will those fat rich people go-- who loot the poor with both hands, and go to the Ganges to wash away their sin, and offer holy water in temples?\"\n",
        "\n",
        "This mood of piety too changed; variability is the special quality of intoxication. It was the turn of despair and grief. Madhav said, \"But the poor thing suffered a great deal in her life. Even her death was so painful!\" Covering his eyes with his hands, he began to weep, [and sobbed loudly].\n",
        "\n",
        "Ghisu consoled him: \"Why do you weep, son? Be happy that she's been liberated from this net of illusion. She's escaped from the snare; she was very fortunate that she was able to break the bonds of worldly illusion so quickly.\"\n",
        "\n",
        "And both, standing there, began to sing, \"Temptress! Why do your eyes flash, temptress?\"\n",
        "\n",
        "The whole wine-house was absorbed in the spectacle, and these two drinkers, deep in intoxication, kept on singing. Then they both began to dance-- they leaped and jumped, fell down, flounced about, gesticulated, [strutted around]; and finally, overcome by drunkenness, they collapsed.\n",
        " \"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFLlUQqhuo0v",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "077b1a81-cdeb-4b5d-eb5d-76a7bdf2ebe0"
      },
      "source": [
        "from nltk.translate.bleu_score import sentence_bleu\n",
        "from nltk.translate.bleu_score import SmoothingFunction\n",
        "from nltk.tokenize import sent_tokenize, word_tokenize\n",
        "nltk.download('punkt')\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qdu_q3c4vgdH"
      },
      "source": [
        " smoothie = SmoothingFunction().method4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AkKRR2uBIl74"
      },
      "source": [
        "score = sentence_bleu([refrence], story, smoothing_function=smoothie)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EiMLS9IhunkZ",
        "outputId": "7ee03d9b-b10a-4b8e-bd9f-712e9f46349c"
      },
      "source": [
        "print(\"BLEU score for the story is :\",score)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BLEU score for the story is : 0.6395698080914985\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w8nRpwMhLqiC",
        "outputId": "2ec2ce0d-0350-4779-8c00-facdf60e04b9"
      },
      "source": [
        "import gensim\n",
        "from gensim.models import Word2Vec"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2021-10-01 11:00:05 | INFO | summarizer.preprocessing.cleaner | 'pattern' package not found; tag filters are not available for English\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GeYjx4XILqrh"
      },
      "source": [
        "data = []\n",
        "for i in sent_tokenize(story):\n",
        "    temp = []\n",
        "    # tokenize the sentence into words\n",
        "    for j in word_tokenize(i):\n",
        "        temp.append(j.lower())\n",
        "    data.append(temp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UBoqeszSLqwz",
        "outputId": "c9e43e98-e828-425b-8403-c98333150e4e"
      },
      "source": [
        "# skip gram model\n",
        "model_1 = gensim.models.Word2Vec(data, min_count = 1,size = 100, window = 5, sg=0) \n",
        "\n",
        "# CBOW \n",
        "model_2 = gensim.models.Word2Vec(data, min_count = 1,size = 100, window = 5, sg=1) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "2021-10-01 11:40:06 | INFO | gensim.models.word2vec | collecting all words and their counts\n",
            "2021-10-01 11:40:06 | INFO | gensim.models.word2vec | PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
            "2021-10-01 11:40:06 | INFO | gensim.models.word2vec | collected 820 word types from a corpus of 3105 raw words and 246 sentences\n",
            "2021-10-01 11:40:06 | INFO | gensim.models.word2vec | Loading a fresh vocabulary\n",
            "2021-10-01 11:40:06 | INFO | gensim.models.word2vec | effective_min_count=1 retains 820 unique words (100% of original 820, drops 0)\n",
            "2021-10-01 11:40:06 | INFO | gensim.models.word2vec | effective_min_count=1 leaves 3105 word corpus (100% of original 3105, drops 0)\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | deleting the raw counts dictionary of 820 items\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | sample=0.001 downsamples 59 most-common words\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | downsampling leaves estimated 2021 word corpus (65.1% of prior 3105)\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | estimated required memory for 820 words and 100 dimensions: 1066000 bytes\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | resetting layer weights\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | training model with 3 workers on 820 vocabulary and 100 features, using sg=0 hs=0 sample=0.001 negative=5 window=5\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 1 : training on 3105 raw words (2000 effective words) took 0.0s, 145915 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 2 : training on 3105 raw words (2023 effective words) took 0.0s, 128513 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 3 : training on 3105 raw words (2029 effective words) took 0.0s, 145457 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 4 : training on 3105 raw words (2005 effective words) took 0.0s, 218810 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 5 : training on 3105 raw words (1999 effective words) took 0.0s, 150983 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | training on a 15525 raw words (10056 effective words) took 0.1s, 95868 effective words/s\n",
            "2021-10-01 11:40:07 | WARNING | gensim.models.base_any2vec | under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | collecting all words and their counts\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | collected 820 word types from a corpus of 3105 raw words and 246 sentences\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | Loading a fresh vocabulary\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | effective_min_count=1 retains 820 unique words (100% of original 820, drops 0)\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | effective_min_count=1 leaves 3105 word corpus (100% of original 3105, drops 0)\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | deleting the raw counts dictionary of 820 items\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | sample=0.001 downsamples 59 most-common words\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | downsampling leaves estimated 2021 word corpus (65.1% of prior 3105)\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | estimated required memory for 820 words and 100 dimensions: 1066000 bytes\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.word2vec | resetting layer weights\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | training model with 3 workers on 820 vocabulary and 100 features, using sg=1 hs=0 sample=0.001 negative=5 window=5\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 1 : training on 3105 raw words (2000 effective words) took 0.0s, 101785 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 2 : training on 3105 raw words (2023 effective words) took 0.0s, 120482 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 3 : training on 3105 raw words (2029 effective words) took 0.0s, 107753 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 4 : training on 3105 raw words (2005 effective words) took 0.0s, 114361 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 2 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 1 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | worker thread finished; awaiting finish of 0 more threads\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | EPOCH - 5 : training on 3105 raw words (1999 effective words) took 0.0s, 137416 effective words/s\n",
            "2021-10-01 11:40:07 | INFO | gensim.models.base_any2vec | training on a 15525 raw words (10056 effective words) took 0.1s, 80129 effective words/s\n",
            "2021-10-01 11:40:07 | WARNING | gensim.models.base_any2vec | under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "poqoD_4nLqzi",
        "outputId": "fe4e031b-6493-45f3-a578-b8d0182cf71a"
      },
      "source": [
        "print(\"Cosine similarity between money and drinking - Skip gram model is\" , model_1.similarity(\"money\",\"drinking\"))\n",
        "\n",
        "# while trying to find cosine similarity between few words I was facing the error which states \"word\" not in vocabulary while that word is surely in the story.\n",
        "# How to deal with it? "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cosine similarity between money and drinking - Skip gram model is -0.048346274\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0E90pKxgPHBc",
        "outputId": "e676841c-3e36-4876-ef20-f10ae51b4e7d"
      },
      "source": [
        "print(\"Cosine similarity between money and drinking - CBOW model is\" , model_2.similarity(\"money\",\"drinking\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cosine similarity between money and drinking - CBOW model is 0.67639065\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "99nvG8_GN3a9",
        "outputId": "bc7d8e57-9627-420d-eb28-f5c5f221dccc"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
            "  \"\"\"Entry point for launching an IPython kernel.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8P86MHH6N3eC"
      },
      "source": [
        "# EMBEDDINGS. 1st layer should be embediing"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VvZ8U3dnvrwz"
      },
      "source": [
        "while True:\n",
        "  pass"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6h-60MeLqBk"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aEeWZnS4OLyJ"
      },
      "source": [
        "import nltk"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "dnJHhaYPQj87",
        "outputId": "ce5741f5-95a4-47f4-b778-47aaec10ea20"
      },
      "source": [
        "nltk.__version__"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'3.2.5'"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    }
  ]
}