Awesome_align_and_PyVi.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Awesome-align and PyVi",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPB1iN1Dgt5mLvfjlSXn/ei",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/quocthang0507/VietnameseNaturalLanguageProcessing/blob/main/Awesome_align_and_PyVi.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SQfp6N7MfGKw"
      },
      "source": [
        "Awesome-align Github repo: https://github.com/neulab/awesome-align\n",
        "\n",
        "Awesome-align Citation:\n",
        "```\n",
        "@inproceedings{dou2021word,\n",
        "  title={Word Alignment by Fine-tuning Embeddings on Parallel Corpora},\n",
        "  author={Dou, Zi-Yi and Neubig, Graham},\n",
        "  booktitle={Conference of the European Chapter of the Association for Computational Linguistics (EACL)},\n",
        "  year={2021}\n",
        "}\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CB-hauM7fYy-"
      },
      "source": [
        "PyVi Github repo: https://github.com/trungtv/pyvi"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e015_n7YjoAs",
        "outputId": "1b272083-fee5-432b-9d0a-13fae8dc663d"
      },
      "source": [
        "!pip3 install pyvi\n",
        "!pip3 install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyvi in /usr/local/lib/python3.7/dist-packages (0.1.1)\n",
            "Requirement already satisfied: sklearn-crfsuite in /usr/local/lib/python3.7/dist-packages (from pyvi) (0.3.6)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from pyvi) (0.22.2.post1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->pyvi) (1.0.1)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->pyvi) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->pyvi) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite->pyvi) (4.41.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite->pyvi) (1.15.0)\n",
            "Requirement already satisfied: python-crfsuite>=0.8.3 in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite->pyvi) (0.9.7)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from sklearn-crfsuite->pyvi) (0.8.9)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.9.2)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (5.4.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: huggingface-hub==0.0.12 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.12)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.5.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IQVkTbU35hEI"
      },
      "source": [
        "# For awesome-align packages\n",
        "import torch\n",
        "import transformers\n",
        "import itertools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FwPSa4Tej95t"
      },
      "source": [
        "# For PyVi package\n",
        "from pyvi import ViTokenizer, ViPosTagger"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WwDxBNxA5qFV",
        "outputId": "2e02c849-eeb8-4034-9193-92a9c6ef4edc"
      },
      "source": [
        "model = transformers.BertModel.from_pretrained('bert-base-multilingual-cased')\n",
        "tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-multilingual-cased')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cg-IAfbM5uKJ"
      },
      "source": [
        "src = 'Bà ta xuất hiện trong nhiều tấm hình lớn ở hiện trường.'\n",
        "tgt = 'She appears in many big photos at the scene.'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0JTa6D2Y5t8"
      },
      "source": [
        "sent_src = [i.replace('_', ' ') for i in ViTokenizer.tokenize(src).split()]\n",
        "sent_tgt = [i.replace('_', ' ') for i in ViTokenizer.tokenize(tgt).split()]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S6SEc1Jq6LRx"
      },
      "source": [
        "token_src, token_tgt = [tokenizer.tokenize(word) for word in sent_src], [tokenizer.tokenize(word) for word in sent_tgt]\n",
        "wid_src, wid_tgt = [tokenizer.convert_tokens_to_ids(x) for x in token_src], [tokenizer.convert_tokens_to_ids(x) for x in token_tgt]\n",
        "ids_src, ids_tgt = tokenizer.prepare_for_model(list(itertools.chain(*wid_src)), return_tensors='pt', model_max_length=tokenizer.model_max_length, truncation=True)['input_ids'], tokenizer.prepare_for_model(list(itertools.chain(*wid_tgt)), return_tensors='pt', truncation=True, model_max_length=tokenizer.model_max_length)['input_ids']\n",
        "sub2word_map_src = []\n",
        "for i, word_list in enumerate(token_src):\n",
        "  sub2word_map_src += [i for x in word_list]\n",
        "sub2word_map_tgt = []\n",
        "for i, word_list in enumerate(token_tgt):\n",
        "  sub2word_map_tgt += [i for x in word_list]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsmHSSZk6Oj5"
      },
      "source": [
        "align_layer = 8\n",
        "threshold = 1e-3\n",
        "model.eval()\n",
        "with torch.no_grad():\n",
        "  out_src = model(ids_src.unsqueeze(0), output_hidden_states=True)[2][align_layer][0, 1:-1]\n",
        "  out_tgt = model(ids_tgt.unsqueeze(0), output_hidden_states=True)[2][align_layer][0, 1:-1]\n",
        "\n",
        "  dot_prod = torch.matmul(out_src, out_tgt.transpose(-1, -2))\n",
        "\n",
        "  softmax_srctgt = torch.nn.Softmax(dim=-1)(dot_prod)\n",
        "  softmax_tgtsrc = torch.nn.Softmax(dim=-2)(dot_prod)\n",
        "\n",
        "  softmax_inter = (softmax_srctgt > threshold)*(softmax_tgtsrc > threshold)\n",
        "\n",
        "align_subwords = torch.nonzero(softmax_inter, as_tuple=False)\n",
        "align_words = set()\n",
        "for i, j in align_subwords:\n",
        "  align_words.add( (sub2word_map_src[i], sub2word_map_tgt[j]) )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rq7dIN986Rpg"
      },
      "source": [
        "class color:\n",
        "   PURPLE = '\\033[95m'\n",
        "   CYAN = '\\033[96m'\n",
        "   DARKCYAN = '\\033[36m'\n",
        "   BLUE = '\\033[94m'\n",
        "   GREEN = '\\033[92m'\n",
        "   YELLOW = '\\033[93m'\n",
        "   RED = '\\033[91m'\n",
        "   BOLD = '\\033[1m'\n",
        "   UNDERLINE = '\\033[4m'\n",
        "   END = '\\033[0m'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LsqSpf4Q6ToZ",
        "outputId": "7b6321a2-7fc4-490f-ce13-78737d1b5b88"
      },
      "source": [
        "for i, j in sorted(align_words):\n",
        "  print(f'{color.BOLD}{color.BLUE}{sent_src[i]}{color.END}==={color.BOLD}{color.RED}{sent_tgt[j]}{color.END}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1m\u001b[94mBà\u001b[0m===\u001b[1m\u001b[91mShe\u001b[0m\n",
            "\u001b[1m\u001b[94mxuất hiện\u001b[0m===\u001b[1m\u001b[91mappears\u001b[0m\n",
            "\u001b[1m\u001b[94mtrong\u001b[0m===\u001b[1m\u001b[91min\u001b[0m\n",
            "\u001b[1m\u001b[94mnhiều\u001b[0m===\u001b[1m\u001b[91mmany\u001b[0m\n",
            "\u001b[1m\u001b[94mhình\u001b[0m===\u001b[1m\u001b[91mphotos\u001b[0m\n",
            "\u001b[1m\u001b[94mlớn\u001b[0m===\u001b[1m\u001b[91mbig\u001b[0m\n",
            "\u001b[1m\u001b[94mở\u001b[0m===\u001b[1m\u001b[91mat\u001b[0m\n",
            "\u001b[1m\u001b[94mhiện trường\u001b[0m===\u001b[1m\u001b[91mthe\u001b[0m\n",
            "\u001b[1m\u001b[94mhiện trường\u001b[0m===\u001b[1m\u001b[91mscene\u001b[0m\n",
            "\u001b[1m\u001b[94m.\u001b[0m===\u001b[1m\u001b[91m.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}