KLUE_DP.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "MGaqyMD52aGa",
      "metadata": {
        "id": "MGaqyMD52aGa"
      },
      "outputs": [],
      "source": [
        "#https://towardsdatascience.com/how-to-create-and-train-a-multi-task-transformer-model-18c54a14624\n",
        "#https://medium.com/@shahrukhx01/multi-task-learning-with-transformers-part-1-multi-prediction-heads-b7001cf014bf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "-FiiDKPiD_Et",
      "metadata": {
        "id": "-FiiDKPiD_Et"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "8e59dc52",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8e59dc52",
        "outputId": "75edd512-63d1-4cd5-dfa7-ce673412fe71"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting seqeval\n",
            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.28.1-py3-none-any.whl (7.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m91.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets\n",
            "  Downloading datasets-2.12.0-py3-none-any.whl (474 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m474.6/474.6 kB\u001b[0m \u001b[31m48.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (1.10.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.22.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.14.1-py3-none-any.whl (224 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m224.5/224.5 kB\u001b[0m \u001b[31m28.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m107.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m79.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m24.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.4.0)\n",
            "Collecting dill<0.3.7,>=0.3.0\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.1.0)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.2.0)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m34.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Building wheels for collected packages: seqeval\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16180 sha256=877ce6fac714fab1833325755284e4718ea3cb3d71c7c62f914a28308e6fc88b\n",
            "  Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",
            "Successfully built seqeval\n",
            "Installing collected packages: tokenizers, xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, huggingface-hub, aiosignal, transformers, seqeval, aiohttp, datasets\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.12.0 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.14.1 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 seqeval-1.2.2 tokenizers-0.13.3 transformers-4.28.1 xxhash-3.2.0 yarl-1.9.2\n"
          ]
        }
      ],
      "source": [
        "!pip install -U seqeval transformers datasets scipy scikit-learn"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "XLLtb_LzJNgW",
      "metadata": {
        "id": "XLLtb_LzJNgW"
      },
      "outputs": [],
      "source": [
        "#from huggingface_hub import notebook_login\n",
        "\n",
        "#notebook_login()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2O8kHxYWJOZE",
      "metadata": {
        "id": "2O8kHxYWJOZE"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "id": "92e868b7-3cf7-4977-a6e7-ebbb99ba298e",
      "metadata": {
        "id": "92e868b7-3cf7-4977-a6e7-ebbb99ba298e"
      },
      "source": [
        "노트북을 실행하는데 필요한 라이브러리들을 모두 임포트합니다."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "K1zw9D2_Qdk6",
      "metadata": {
        "id": "K1zw9D2_Qdk6"
      },
      "outputs": [],
      "source": [
        "import logging\n",
        "import os\n",
        "import random\n",
        "import sys\n",
        "from dataclasses import dataclass, field\n",
        "from typing import Optional\n",
        "\n",
        "import datasets\n",
        "import numpy as np\n",
        "from datasets import load_dataset\n",
        "\n",
        "import transformers\n",
        "from transformers import (\n",
        "    AutoConfig,\n",
        "    AutoModelForSequenceClassification,\n",
        "    AutoTokenizer,\n",
        "    DataCollatorWithPadding,\n",
        "    DataCollatorForTokenClassification,\n",
        "    EvalPrediction,\n",
        "    HfArgumentParser,\n",
        "    PretrainedConfig,\n",
        "    Trainer,\n",
        "    TrainingArguments,\n",
        "    default_data_collator,\n",
        "    set_seed,\n",
        ")\n",
        "from transformers.trainer_utils import get_last_checkpoint\n",
        "from transformers.utils import check_min_version, send_example_telemetry\n",
        "from transformers.utils.versions import require_version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "243b461f",
      "metadata": {
        "id": "243b461f"
      },
      "outputs": [],
      "source": [
        "\n",
        "import pandas as pd\n",
        "\n",
        "from datasets import load_dataset, load_metric, ClassLabel, Sequence\n",
        "# Will error if the minimal version of Transformers is not installed. Remove at your own risks.\n",
        "\n",
        "\n",
        "task_to_keys = {\n",
        "    \"cola\": (\"sentence\", None),\n",
        "    \"mnli\": (\"premise\", \"hypothesis\"),\n",
        "    \"mrpc\": (\"sentence1\", \"sentence2\"),\n",
        "    \"qnli\": (\"question\", \"sentence\"),\n",
        "    \"qqp\": (\"question1\", \"question2\"),\n",
        "    \"rte\": (\"sentence1\", \"sentence2\"),\n",
        "    \"sst2\": (\"sentence\", None),\n",
        "    \"stsb\": (\"sentence1\", \"sentence2\"),\n",
        "    \"wnli\": (\"sentence1\", \"sentence2\"),\n",
        "}\n",
        "\n",
        "logger = logging.getLogger(__name__)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "aLuW1oq3beic",
      "metadata": {
        "id": "aLuW1oq3beic"
      },
      "outputs": [],
      "source": [
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "9mO73fRNcYPP",
      "metadata": {
        "id": "9mO73fRNcYPP"
      },
      "outputs": [],
      "source": [
        "import argparse\n",
        "import logging\n",
        "import os\n",
        "from typing import Any, List, Optional, Tuple\n",
        "from torch.utils.data import DataLoader, TensorDataset\n",
        "from transformers import PreTrainedModel,AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, AutoModel, PreTrainedTokenizer\n",
        "from torch import nn\n",
        "import torch\n",
        "from transformers.modeling_outputs import TokenClassifierOutput\n",
        "from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence\n",
        "import torch.nn.functional as F\n",
        "from torch.nn.parameter import Parameter\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "rqMZLl0HRAnI",
      "metadata": {
        "id": "rqMZLl0HRAnI"
      },
      "outputs": [],
      "source": [
        "\n",
        "@dataclass\n",
        "class DataTrainingArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to what data we are going to input our model for training and eval.\n",
        "    Using `HfArgumentParser` we can turn this class\n",
        "    into argparse arguments to be able to specify them on\n",
        "    the command line.\n",
        "    \"\"\"\n",
        "\n",
        "    task_name: Optional[str] = field(\n",
        "        default=None,\n",
        "        metadata={\"help\": \"The name of the task to train on: \" + \", \".join(task_to_keys.keys())},\n",
        "    )\n",
        "    dataset_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"The name of the dataset to use (via the datasets library).\"}\n",
        "    )\n",
        "    dataset_config_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"The configuration name of the dataset to use (via the datasets library).\"}\n",
        "    )\n",
        "    max_seq_length: int = field(\n",
        "        default=128,\n",
        "        metadata={\n",
        "            \"help\": (\n",
        "                \"The maximum total input sequence length after tokenization. Sequences longer \"\n",
        "                \"than this will be truncated, sequences shorter will be padded.\"\n",
        "            )\n",
        "        },\n",
        "    )\n",
        "    overwrite_cache: bool = field(\n",
        "        default=False, metadata={\"help\": \"Overwrite the cached preprocessed datasets or not.\"}\n",
        "    )\n",
        "    pad_to_max_length: bool = field(\n",
        "        default=True,\n",
        "        metadata={\n",
        "            \"help\": (\n",
        "                \"Whether to pad all samples to `max_seq_length`. \"\n",
        "                \"If False, will pad the samples dynamically when batching to the maximum length in the batch.\"\n",
        "            )\n",
        "        },\n",
        "    )\n",
        "    max_train_samples: Optional[int] = field(\n",
        "        default=None,\n",
        "        metadata={\n",
        "            \"help\": (\n",
        "                \"For debugging purposes or quicker training, truncate the number of training examples to this \"\n",
        "                \"value if set.\"\n",
        "            )\n",
        "        },\n",
        "    )\n",
        "    max_eval_samples: Optional[int] = field(\n",
        "        default=None,\n",
        "        metadata={\n",
        "            \"help\": (\n",
        "                \"For debugging purposes or quicker training, truncate the number of evaluation examples to this \"\n",
        "                \"value if set.\"\n",
        "            )\n",
        "        },\n",
        "    )\n",
        "    max_predict_samples: Optional[int] = field(\n",
        "        default=None,\n",
        "        metadata={\n",
        "            \"help\": (\n",
        "                \"For debugging purposes or quicker training, truncate the number of prediction examples to this \"\n",
        "                \"value if set.\"\n",
        "            )\n",
        "        },\n",
        "    )\n",
        "    train_file: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"A tsv or a json file containing the training data.\"}\n",
        "    )\n",
        "    validation_file: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"A tsv or a json file containing the validation data.\"}\n",
        "    )\n",
        "    test_file: Optional[str] = field(default=None, metadata={\"help\": \"A tsv or a json file containing the test data.\"})\n",
        "\n",
        "    def __post_init__(self):\n",
        "        if self.task_name is not None:\n",
        "            self.task_name = self.task_name.lower()\n",
        "            if self.task_name not in task_to_keys.keys():\n",
        "                raise ValueError(\"Unknown task, you should pick one in \" + \",\".join(task_to_keys.keys()))\n",
        "        elif self.dataset_name is not None:\n",
        "            pass\n",
        "        elif self.train_file is None or self.validation_file is None:\n",
        "            raise ValueError(\"Need either a GLUE task, a training/validation file or a dataset name.\")\n",
        "        else:\n",
        "            train_extension = self.train_file.split(\".\")[-1]\n",
        "            assert train_extension in [\"tsv\", \"json\"], \"`train_file` should be a csv or a json file.\"\n",
        "            validation_extension = self.validation_file.split(\".\")[-1]\n",
        "            assert (\n",
        "                validation_extension == train_extension\n",
        "            ), \"`validation_file` should have the same extension (tsv or json) as `train_file`.\"\n",
        "\n",
        "\n",
        "@dataclass\n",
        "class ModelArguments:\n",
        "    \"\"\"\n",
        "    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.\n",
        "    \"\"\"\n",
        "\n",
        "    encoder_name_or_path: str = field(\n",
        "        metadata={\"help\": \"Path to pretrained model or model identifier from huggingface.co/models\"}\n",
        "    )\n",
        "    config_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained config name or path if not the same as model_name\"}\n",
        "    )\n",
        "    tokenizer_name: Optional[str] = field(\n",
        "        default=None, metadata={\"help\": \"Pretrained tokenizer name or path if not the same as model_name\"}\n",
        "    )\n",
        "    cache_dir: Optional[str] = field(\n",
        "        default=None,\n",
        "        metadata={\"help\": \"Where do you want to store the pretrained models downloaded from huggingface.co\"},\n",
        "    )\n",
        "    use_fast_tokenizer: bool = field(\n",
        "        default=True,\n",
        "        metadata={\"help\": \"Whether to use one of the fast tokenizer (backed by the tokenizers library) or not.\"},\n",
        "    )\n",
        "    model_revision: str = field(\n",
        "        default=\"main\",\n",
        "        metadata={\"help\": \"The specific model version to use (can be a branch name, tag name or commit id).\"},\n",
        "    )\n",
        "    use_auth_token: bool = field(\n",
        "        default=False,\n",
        "        metadata={\n",
        "            \"help\": (\n",
        "                \"Will use the token generated when running `huggingface-cli login` (necessary to use this script \"\n",
        "                \"with private models).\"\n",
        "            )\n",
        "        },\n",
        "    )\n",
        "    ignore_mismatched_sizes: bool = field(\n",
        "        default=False,\n",
        "        metadata={\"help\": \"Will enable to load a pretrained model whose head dimensions are different.\"},\n",
        "    )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "xFdbE5ngblIZ",
      "metadata": {
        "id": "xFdbE5ngblIZ"
      },
      "outputs": [],
      "source": [
        "def load_datasets(dataprocessor, data_args):\n",
        "\n",
        "    train_dataset = dataprocessor._create_dataset(data_args.train_file,'train')\n",
        "    validation_dataset = dataprocessor._create_dataset(data_args.validation_file,'validation')\n",
        "\n",
        "    head_task = Task(\n",
        "                    id=0,\n",
        "                    name=\"head_id\",\n",
        "                    num_labels=40,\n",
        "                    type=\"token_classification\",\n",
        "                    )\n",
        "\n",
        "    dep_task = Task(\n",
        "                    id=1,\n",
        "                    name=\"dep_ids\",\n",
        "                    num_labels=38,\n",
        "                    type=\"token_classification\",\n",
        "                    )\n",
        "    \n",
        "\n",
        "    dataset = datasets.DatasetDict(\n",
        "        {\"train\": train_dataset, \"validation\": validation_dataset}\n",
        "    )\n",
        "    tasks = [head_task, dep_task]\n",
        "    return tasks, dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "WpI4Wbue7tBS",
      "metadata": {
        "id": "WpI4Wbue7tBS"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "J3MyHkOJaZEB",
      "metadata": {
        "id": "J3MyHkOJaZEB"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "yPGGqqK6auIf",
      "metadata": {
        "id": "yPGGqqK6auIf"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "3HKICAKJaZYZ",
      "metadata": {
        "id": "3HKICAKJaZYZ"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "lNl2R1n3O6bH",
      "metadata": {
        "id": "lNl2R1n3O6bH"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "id": "59b1e5ae-471f-461f-850f-4241a6037471",
      "metadata": {
        "id": "59b1e5ae-471f-461f-850f-4241a6037471"
      },
      "source": [
        "학습에 필요한 정보를 변수로 기록합니다.\n",
        "\n",
        "본 노트북에서는 `klue-roberta-base` 모델을 활용하지만, https://huggingface.co/klue 페이지에서 더 다양한 사전학습 언어 모델을 확인하실 수 있습니다.\n",
        "\n",
        "학습 태스크로는 `nli`를, 배치 사이즈로는 32를 지정하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "MTUg-FT9JRwN",
      "metadata": {
        "id": "MTUg-FT9JRwN"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "ub-XuTr8Rq75",
      "metadata": {
        "id": "ub-XuTr8Rq75"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "id": "Aoz2sN8SZct3",
      "metadata": {
        "id": "Aoz2sN8SZct3"
      },
      "outputs": [],
      "source": [
        "def get_dep_labels() -> List[str]:\n",
        "    \"\"\"\n",
        "    label for dependency relations format:\n",
        "    {structure}_(optional){function}\n",
        "    \"\"\"\n",
        "    dep_labels = [\n",
        "        \"NP\",\n",
        "        \"NP_AJT\",\n",
        "        \"VP\",\n",
        "        \"NP_SBJ\",\n",
        "        \"VP_MOD\",\n",
        "        \"NP_OBJ\",\n",
        "        \"AP\",\n",
        "        \"NP_CNJ\",\n",
        "        \"NP_MOD\",\n",
        "        \"VNP\",\n",
        "        \"DP\",\n",
        "        \"VP_AJT\",\n",
        "        \"VNP_MOD\",\n",
        "        \"NP_CMP\",\n",
        "        \"VP_SBJ\",\n",
        "        \"VP_CMP\",\n",
        "        \"VP_OBJ\",\n",
        "        \"VNP_CMP\",\n",
        "        \"AP_MOD\",\n",
        "        \"X_AJT\",\n",
        "        \"VP_CNJ\",\n",
        "        \"VNP_AJT\",\n",
        "        \"IP\",\n",
        "        \"X\",\n",
        "        \"X_SBJ\",\n",
        "        \"VNP_OBJ\",\n",
        "        \"VNP_SBJ\",\n",
        "        \"X_OBJ\",\n",
        "        \"AP_AJT\",\n",
        "        \"L\",\n",
        "        \"X_MOD\",\n",
        "        \"X_CNJ\",\n",
        "        \"VNP_CNJ\",\n",
        "        \"X_CMP\",\n",
        "        \"AP_CMP\",\n",
        "        \"AP_SBJ\",\n",
        "        \"R\",\n",
        "        \"NP_SVJ\",\n",
        "    ]\n",
        "    return dep_labels\n",
        "\n",
        "\n",
        "def get_pos_labels() -> List[str]:\n",
        "    \"\"\"label for part-of-speech tags\"\"\"\n",
        "\n",
        "    return [\n",
        "        \"NNG\",\n",
        "        \"NNP\",\n",
        "        \"NNB\",\n",
        "        \"NP\",\n",
        "        \"NR\",\n",
        "        \"VV\",\n",
        "        \"VA\",\n",
        "        \"VX\",\n",
        "        \"VCP\",\n",
        "        \"VCN\",\n",
        "        \"MMA\",\n",
        "        \"MMD\",\n",
        "        \"MMN\",\n",
        "        \"MAG\",\n",
        "        \"MAJ\",\n",
        "        \"JC\",\n",
        "        \"IC\",\n",
        "        \"JKS\",\n",
        "        \"JKC\",\n",
        "        \"JKG\",\n",
        "        \"JKO\",\n",
        "        \"JKB\",\n",
        "        \"JKV\",\n",
        "        \"JKQ\",\n",
        "        \"JX\",\n",
        "        \"EP\",\n",
        "        \"EF\",\n",
        "        \"EC\",\n",
        "        \"ETN\",\n",
        "        \"ETM\",\n",
        "        \"XPN\",\n",
        "        \"XSN\",\n",
        "        \"XSV\",\n",
        "        \"XSA\",\n",
        "        \"XR\",\n",
        "        \"SF\",\n",
        "        \"SP\",\n",
        "        \"SS\",\n",
        "        \"SE\",\n",
        "        \"SO\",\n",
        "        \"SL\",\n",
        "        \"SH\",\n",
        "        \"SW\",\n",
        "        \"SN\",\n",
        "        \"NA\",\n",
        "    ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "6ecaacbe",
      "metadata": {
        "id": "6ecaacbe"
      },
      "outputs": [],
      "source": [
        "@dataclass\n",
        "class Task:\n",
        "    id: int\n",
        "    name: str\n",
        "    type: str\n",
        "    num_labels: int"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "816b5893-d03e-4c6e-8536-e246b1def987",
      "metadata": {
        "id": "816b5893-d03e-4c6e-8536-e246b1def987"
      },
      "source": [
        "이제 HuggingFace `datasets` 라이브러리에 등록된 KLUE 데이터셋 중, NLI 데이터를 내려받습니다."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "f8feeba0",
      "metadata": {
        "id": "f8feeba0"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "id": "f90b06e4-3795-435a-aec8-701da9bd5f04",
      "metadata": {
        "id": "f90b06e4-3795-435a-aec8-701da9bd5f04"
      },
      "source": [
        "다운로드 혹은 로드 후 얻어진 `datasets` 객체를 살펴보면, 훈련 데이터와 검증 데이터가 포함되어 있는 것을 확인할 수 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "-O56ZQITxLp1",
      "metadata": {
        "id": "-O56ZQITxLp1"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "id": "18slzdARbSfx",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "abd1080af38a4e2894cf2a5cfa06fae4",
            "74c273e7e7b64242adc6004def38b843",
            "edf3c9b38ba74ed1b2f93255c70882ba",
            "93a50faab46a40839612d729abea8fae",
            "310adfbc64b644c8b454a5d10cd30c39",
            "abd98c227656448ca9a2c231f7d857d6",
            "315211d6895c4d198b23f8ccf873b808",
            "620a6da08dbc4f0d9237737675b16b07",
            "8b1fe2103510434cbd8476c41648a043",
            "ea75731db26f40aeb96f3810f4d1b1c1",
            "f4c43edd31a748ad998e3d80d0ce04d9",
            "7975735bb20a4eff83e80edccaec76cc",
            "7abb2f60581b4a83bd86183d635cf70f",
            "64251bda5720477b92415862eb0be923",
            "0fca6111c86c439d9d1956c66c9d8bb4",
            "f044aa8dcc7d447eb3decafa17ee8048",
            "9346162673844c39bf3f17f12a36323c",
            "f2e976a2d7c249d5b4c76486ff3f66fc",
            "dc6aabeeabbf4b63b1a038c98ee298ff",
            "5f718d11d839434a88bf8ff10f6dba8d",
            "150121e1ef8d4b62a698c857552b9c4d",
            "eb27f6e1353c48318d075be6812799f3",
            "c89a9979474047838265db9e5b174bbd",
            "9614bd4a9a54499b95adaa3086ac19bb",
            "a20fdcd07a5d45b184f420650064a2e7",
            "67a8c60e2723447e8b07b754d3d8ae56",
            "d5fcd237b1ab4e56b0ba26c66cea29ea",
            "5ea4b2d359394760b53ebbe3fc284ef2",
            "219670845a3a42149f8d534a9d91a4ea",
            "5947c0fa58d44381a92be91b117dae5f",
            "85557921902146b783ddf1eb4f81cf1f",
            "e9dfbb1dbddd44968c939048dde40b45",
            "1bdccc0213cc453bb20e5e4abfd32e35",
            "7426e533f0fa47e1b984b64579567eb3",
            "37308a8e76214ccc9f2cbef04caa38eb",
            "a2edded1f3fb4a7fa5e56f061d2a66a0",
            "011071963b2d4673bcc64a1bb1563946",
            "adec0539c4b7498eb6ff0a13c4d088ef",
            "50d082a0c56a434da92d46b06faf4183",
            "f2e571828f4945bf9285bf4de06cb886",
            "506b8bd646eb4a86b45785fca7854980",
            "556e9f565c904d27a1be0d97d0f7815e",
            "1b464d83af294cfd975133ccfafafb68",
            "b34136ec3fba4f539478a9bddb035bde",
            "318a47c48d7149cbb5aaae5328f3a3c0",
            "0df8bf6a7b21437593611d71e0b070eb",
            "13f4a07998da4904a092831457750a5d",
            "3e469d02ea9f4c37ae0b4da4c7aeab8e",
            "e7219bb4c46a4cb4b70ea67a4c6e3107",
            "e47c0c2f4d1e4605907c5828e0744c5e",
            "f9f64e83e6024bf38ee31dfaf0aa8f64",
            "7dbf3262548c4087801c93f8a7df5e09",
            "48faa8b6ebaf4a38beee79d8f8561297",
            "bc1467abf51245b7a8b9f1745365893d",
            "1155b6a9e1b042d8b91d11b34f649954"
          ]
        },
        "id": "18slzdARbSfx",
        "outputId": "1137df34-2702-4b39-ae60-2b009b5a4f20"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "abd1080af38a4e2894cf2a5cfa06fae4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/289 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7975735bb20a4eff83e80edccaec76cc",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/425 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c89a9979474047838265db9e5b174bbd",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/248k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7426e533f0fa47e1b984b64579567eb3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/495k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "318a47c48d7149cbb5aaae5328f3a3c0",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "data_args = DataTrainingArguments(train_file = \"/content/klue-dp-v1.1_train.tsv\", validation_file = \"/content/klue-dp-v1.1_dev.tsv\", max_seq_length = 128)\n",
        "model_args = ModelArguments(encoder_name_or_path=\"klue/bert-base\")\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\n",
        "    model_args.encoder_name_or_path,\n",
        "    cache_dir=model_args.cache_dir,\n",
        "    use_fast=model_args.use_fast_tokenizer,\n",
        "    revision=model_args.model_revision,\n",
        "    use_auth_token=True if model_args.use_auth_token else None,\n",
        ")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "id": "xI7wpecqb_wY",
      "metadata": {
        "id": "xI7wpecqb_wY"
      },
      "outputs": [],
      "source": [
        "\n",
        "class KlueDPInputExample:\n",
        "    \"\"\"A single training/test example for Dependency Parsing in .conllu format\n",
        "    Args:\n",
        "        guid : Unique id for the example\n",
        "        text : string. the original form of sentence\n",
        "        token_id : token id\n",
        "        token : 어절\n",
        "        pos : POS tag(s)\n",
        "        head : dependency head\n",
        "        dep : dependency relation\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self, guid: str, text: str, sent_id: int, token_id: int, token: str, pos: str, head: str, dep: str\n",
        "    ) -> None:\n",
        "        self.guid = guid\n",
        "        self.text = text\n",
        "        self.sent_id = sent_id\n",
        "        self.token_id = token_id\n",
        "        self.token = token\n",
        "        self.pos = pos\n",
        "        self.head = head\n",
        "        self.dep = dep\n",
        "\n",
        "\n",
        "class KlueDPInputFeatures:\n",
        "    \"\"\"A single set of features of data. Property names are the same names as the corresponding inputs to a model.\n",
        "    Args:\n",
        "        input_ids: Indices of input sequence tokens in the vocabulary.\n",
        "        attention_mask: Mask to avoid performing attention on padding token indices.\n",
        "            Mask values selected in ``[0, 1]``: Usually ``1`` for tokens that are NOT MASKED, ``0`` for MASKED (padded)\n",
        "            tokens.\n",
        "        bpe_head_mask : Mask to mark the head token of bpe in aejeol\n",
        "        head_labels : head ids for each aejeols on head token index\n",
        "        dep_labels : dependecy relations for each aejeols on head token index\n",
        "        pos_ids : pos tag for each aejeols on head token index\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        guid: str,\n",
        "        ids: List[int],\n",
        "        mask: List[int],\n",
        "        bpe_head_mask: List[int],\n",
        "        bpe_tail_mask: List[int],\n",
        "        head_labels: List[int],\n",
        "        dep_labels: List[int],\n",
        "        pos_ids: List[int],\n",
        "    ) -> None:\n",
        "        self.guid = guid\n",
        "        self.input_ids = ids\n",
        "        self.attention_mask = mask\n",
        "        self.bpe_head_mask = bpe_head_mask\n",
        "        self.bpe_tail_mask = bpe_tail_mask\n",
        "        self.head_labels = head_labels\n",
        "        self.dep_labels = dep_labels\n",
        "        self.pos_ids = pos_ids\n",
        "\n",
        "\n",
        "class KlueDPProcessor:\n",
        "\n",
        "    origin_train_file_name = \"klue-dp-v1.1_train.tsv\"\n",
        "    origin_dev_file_name = \"klue-dp-v1.1_dev.tsv\"\n",
        "    origin_test_file_name = \"klue-dp-v1.1_test.tsv\"\n",
        "\n",
        "\n",
        "    def __init__(self, max_seq_length: int, tokenizer: PreTrainedTokenizer) -> None:\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_seq_length = max_seq_length\n",
        "\n",
        "    def _create_examples(self, file_path: str, dataset_type: str) -> List[KlueDPInputExample]:\n",
        "        sent_id = -1\n",
        "        examples = []\n",
        "        with open(file_path, \"r\", encoding=\"utf-8\") as f:\n",
        "            for line in f:\n",
        "                line = line.strip()\n",
        "                if line == \"\" or line == \"\\n\" or line == \"\\t\":\n",
        "                    continue\n",
        "                if line.startswith(\"#\"):\n",
        "                    parsed = line.strip().split(\"\\t\")\n",
        "                    if len(parsed) != 2:  # metadata line about dataset\n",
        "                        continue\n",
        "                    else:\n",
        "                        sent_id += 1\n",
        "                        text = parsed[1].strip()\n",
        "                        guid = parsed[0].replace(\"##\", \"\").strip()\n",
        "                else:\n",
        "                    token_list = [token.replace(\"\\n\", \"\") for token in line.split(\"\\t\")] + [\"-\", \"-\"]\n",
        "                    examples.append(\n",
        "                        KlueDPInputExample(\n",
        "                            guid=guid,\n",
        "                            text=text,\n",
        "                            sent_id=sent_id,\n",
        "                            token_id=int(token_list[0]),\n",
        "                            token=token_list[1],\n",
        "                            pos=token_list[3],\n",
        "                            head=token_list[4],\n",
        "                            dep=token_list[5],\n",
        "                        )\n",
        "                    )\n",
        "        return examples\n",
        "\n",
        "    def convert_examples_to_features(\n",
        "        self,\n",
        "        examples: List[KlueDPInputExample],\n",
        "        tokenizer: PreTrainedTokenizer,\n",
        "        max_length: int,\n",
        "        pos_label_list: List[str],\n",
        "        dep_label_list: List[str],\n",
        "    ) -> List[KlueDPInputFeatures]:\n",
        "\n",
        "        pos_label_map = {label: i for i, label in enumerate(pos_label_list)}\n",
        "        dep_label_map = {label: i for i, label in enumerate(dep_label_list)}\n",
        "\n",
        "        SENT_ID = 0\n",
        "\n",
        "        token_list: List[str] = []\n",
        "        pos_list: List[str] = []\n",
        "        head_list: List[int] = []\n",
        "        dep_list: List[str] = []\n",
        "\n",
        "        features = []\n",
        "        for example in examples:\n",
        "            if SENT_ID != example.sent_id:\n",
        "                SENT_ID = example.sent_id\n",
        "                encoded = tokenizer.encode_plus(\n",
        "                    \" \".join(token_list),\n",
        "                    None,\n",
        "                    add_special_tokens=True,\n",
        "                    max_length=max_length,\n",
        "                    truncation=True,\n",
        "                    padding=\"max_length\",\n",
        "                )\n",
        "\n",
        "                ids, mask = encoded[\"input_ids\"], encoded[\"attention_mask\"]\n",
        "\n",
        "                bpe_head_mask = [0]\n",
        "                bpe_tail_mask = [0]\n",
        "                head_labels = [-100]\n",
        "                dep_labels = [-100]\n",
        "                pos_ids = [-100]  # --> CLS token\n",
        "\n",
        "                for token, head, dep, pos in zip(token_list, head_list, dep_list, pos_list):\n",
        "                    bpe_len = len(tokenizer.tokenize(token))\n",
        "                    head_token_mask = [1] + [0] * (bpe_len - 1)\n",
        "                    tail_token_mask = [0] * (bpe_len - 1) + [1]\n",
        "                    bpe_head_mask.extend(head_token_mask)\n",
        "                    bpe_tail_mask.extend(tail_token_mask)\n",
        "\n",
        "                    head_mask = [head] + [-100] * (bpe_len - 1)\n",
        "                    head_labels.extend(head_mask)\n",
        "                    dep_mask = [dep_label_map[dep]] + [-100] * (bpe_len - 1)\n",
        "                    dep_labels.extend(dep_mask)\n",
        "                    pos_mask = [pos_label_map[pos]] + [-100] * (bpe_len - 1)\n",
        "                    pos_ids.extend(pos_mask)\n",
        "\n",
        "                bpe_head_mask.append(0)\n",
        "                bpe_tail_mask.append(0)\n",
        "                head_labels.append(-100)\n",
        "                dep_labels.append(-100)\n",
        "                pos_ids.append(-100)  # END token\n",
        "                if len(bpe_head_mask) > max_length:\n",
        "                    bpe_head_mask = bpe_head_mask[:max_length]\n",
        "                    bpe_tail_mask = bpe_tail_mask[:max_length]\n",
        "                    head_labels = head_labels[:max_length]\n",
        "                    dep_labels = dep_labels[:max_length]\n",
        "                    pos_ids = pos_ids[:max_length]\n",
        "\n",
        "                else:\n",
        "                    bpe_head_mask.extend([0] * (max_length - len(bpe_head_mask)))  # padding by max_len\n",
        "                    bpe_tail_mask.extend([0] * (max_length - len(bpe_tail_mask)))  # padding by max_len\n",
        "                    head_labels.extend([-100] * (max_length - len(head_labels)))  # padding by max_len\n",
        "                    dep_labels.extend([-100] * (max_length - len(dep_labels)))  # padding by max_len\n",
        "                    pos_ids.extend([-100] * (max_length - len(pos_ids)))\n",
        "\n",
        "                feature = KlueDPInputFeatures(\n",
        "                    guid=example.guid,\n",
        "                    ids=ids,\n",
        "                    mask=mask,\n",
        "                    bpe_head_mask=bpe_head_mask,\n",
        "                    bpe_tail_mask=bpe_tail_mask,\n",
        "                    head_labels=head_labels,\n",
        "                    dep_labels=dep_labels,\n",
        "                    pos_ids=pos_ids,\n",
        "                )\n",
        "                features.append(feature)\n",
        "\n",
        "                token_list = []\n",
        "                pos_list = []\n",
        "                head_list = []\n",
        "                dep_list = []\n",
        "\n",
        "            token_list.append(example.token)\n",
        "            pos_list.append(example.pos.split(\"+\")[-1])  # 맨 뒤 pos정보만 사용\n",
        "            head_list.append(int(example.head))\n",
        "            dep_list.append(example.dep)\n",
        "\n",
        "        encoded = tokenizer.encode_plus(\n",
        "            \" \".join(token_list),\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_length,\n",
        "            truncation=True,\n",
        "            padding=\"max_length\",\n",
        "        )\n",
        "\n",
        "        ids, mask = encoded[\"input_ids\"], encoded[\"attention_mask\"]\n",
        "\n",
        "        bpe_head_mask = [0]\n",
        "        bpe_tail_mask = [0]\n",
        "        head_labels = [-100]\n",
        "        dep_labels = [-100]\n",
        "        pos_ids = [-100]  # --> CLS token\n",
        "\n",
        "        for token, head, dep, pos in zip(token_list, head_list, dep_list, pos_list):\n",
        "            bpe_len = len(tokenizer.tokenize(token))\n",
        "            head_token_mask = [1] + [0] * (bpe_len - 1)\n",
        "            tail_token_mask = [0] * (bpe_len - 1) + [1]\n",
        "            bpe_head_mask.extend(head_token_mask)\n",
        "            bpe_tail_mask.extend(tail_token_mask)\n",
        "\n",
        "            head_mask = [head] + [-100] * (bpe_len - 1)\n",
        "            head_labels.extend(head_mask)\n",
        "            dep_mask = [dep_label_map[dep]] + [-100] * (bpe_len - 1)\n",
        "            dep_labels.extend(dep_mask)\n",
        "            pos_mask = [pos_label_map[pos]] + [-100] * (bpe_len - 1)\n",
        "            pos_ids.extend(pos_mask)\n",
        "\n",
        "        bpe_head_mask.append(0)\n",
        "        bpe_tail_mask.append(0)\n",
        "        head_labels.append(-100)\n",
        "        dep_labels.append(-100)  # END token\n",
        "        bpe_head_mask.extend([0] * (max_length - len(bpe_head_mask)))  # padding by max_len\n",
        "        bpe_tail_mask.extend([0] * (max_length - len(bpe_tail_mask)))  # padding by max_len\n",
        "        head_labels.extend([-100] * (max_length - len(head_labels)))  # padding by max_len\n",
        "        dep_labels.extend([-100] * (max_length - len(dep_labels)))  # padding by max_len\n",
        "        pos_ids.extend([-100] * (max_length - len(pos_ids)))\n",
        "\n",
        "        feature = KlueDPInputFeatures(\n",
        "            guid=example.guid,\n",
        "            ids=ids,\n",
        "            mask=mask,\n",
        "            bpe_head_mask=bpe_head_mask,\n",
        "            bpe_tail_mask=bpe_tail_mask,\n",
        "            head_labels=head_labels,\n",
        "            dep_labels=dep_labels,\n",
        "            pos_ids=pos_ids,\n",
        "        )\n",
        "        features.append(feature)\n",
        "\n",
        "        for feature in features[:3]:\n",
        "            logger.info(\"*** Example ***\")\n",
        "            logger.info(\"input_ids: %s\" % feature.input_ids)\n",
        "            logger.info(\"attention_mask: %s\" % feature.attention_mask)\n",
        "            logger.info(\"bpe_head_mask: %s\" % feature.bpe_head_mask)\n",
        "            logger.info(\"bpe_tail_mask: %s\" % feature.bpe_tail_mask)\n",
        "            logger.info(\"head_id: %s\" % feature.head_labels)\n",
        "            logger.info(\"dep_labels: %s\" % feature.dep_labels)\n",
        "            logger.info(\"pos_ids: %s\" % feature.pos_ids)\n",
        "\n",
        "        return features\n",
        "\n",
        "    def _convert_features(self, examples: List[KlueDPInputExample]) -> List[KlueDPInputFeatures]:\n",
        "        return self.convert_examples_to_features(\n",
        "            examples,\n",
        "            self.tokenizer,\n",
        "            max_length=self.max_seq_length,\n",
        "            dep_label_list=get_dep_labels(),\n",
        "            pos_label_list=get_pos_labels(),\n",
        "        )\n",
        "\n",
        "    def _create_dataset(self, file_path: str, dataset_type: str) -> TensorDataset:\n",
        "        examples = self._create_examples(file_path, dataset_type)\n",
        "        features = self._convert_features(examples)\n",
        "\n",
        "        all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
        "        all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)\n",
        "        all_bpe_head_mask = torch.tensor([f.bpe_head_mask for f in features], dtype=torch.long)\n",
        "        all_bpe_tail_mask = torch.tensor([f.bpe_tail_mask for f in features], dtype=torch.long)\n",
        "        all_head_labels = torch.tensor([f.head_labels for f in features], dtype=torch.long)\n",
        "        all_dep_labels = torch.tensor([f.dep_labels for f in features], dtype=torch.long)\n",
        "        all_pos_ids = torch.tensor([f.pos_ids for f in features], dtype=torch.long)\n",
        "\n",
        "        return TensorDataset(\n",
        "            all_input_ids,\n",
        "            all_attention_mask,\n",
        "            all_bpe_head_mask,\n",
        "            all_bpe_tail_mask,\n",
        "            all_head_labels,\n",
        "            all_dep_labels,\n",
        "            all_pos_ids,\n",
        "        )\n",
        "\n",
        "    def collate_fn(self, batch: List[Tuple]) -> Tuple[torch.Tensor, Any, Any, Any]:\n",
        "        # 1. set args\n",
        "        batch_size = len(batch)\n",
        "        #pos_padding_idx = None if self.hparams.no_pos else len(get_pos_labels())\n",
        "        pos_padding_idx = len(get_pos_labels())\n",
        "        # 2. build inputs : input_ids, attention_mask, bpe_head_mask, bpe_tail_mask\n",
        "        batch_input_ids = []\n",
        "        batch_attention_masks = []\n",
        "        batch_bpe_head_masks = []\n",
        "        batch_bpe_tail_masks = []\n",
        "        batch_token_head_labels = []\n",
        "        batch_token_type_labels = []\n",
        "        for batch_id in range(batch_size):\n",
        "            (\n",
        "                input_id,\n",
        "                attention_mask,\n",
        "                bpe_head_mask,\n",
        "                bpe_tail_mask,\n",
        "                token_head_labels,\n",
        "                token_type_labels,\n",
        "                _,\n",
        "            ) = batch[batch_id]\n",
        "            batch_input_ids.append(input_id)\n",
        "            batch_attention_masks.append(attention_mask)\n",
        "            batch_bpe_head_masks.append(bpe_head_mask)\n",
        "            batch_bpe_tail_masks.append(bpe_tail_mask)\n",
        "            batch_token_head_labels.append(token_head_labels)\n",
        "            batch_token_type_labels.append(token_type_labels)\n",
        "        # 2. build inputs : packing tensors\n",
        "        # 나는 밥을 먹는다. => [CLS] 나 ##는 밥 ##을 먹 ##는 ##다 . [SEP]\n",
        "        # input_id : [2, 717, 2259, 1127, 2069, 1059, 2259, 2062, 18, 3, 0, 0, ...]\n",
        "        # bpe_head_mask : [0, 1, 0, 1, 0, 1, 0, 0, 0, 0, ...] (indicate word start (head) idx)\n",
        "        input_ids = torch.stack(batch_input_ids)\n",
        "        attention_masks = torch.stack(batch_attention_masks)\n",
        "        bpe_head_masks = torch.stack(batch_bpe_head_masks)\n",
        "        bpe_tail_masks = torch.stack(batch_bpe_tail_masks)\n",
        "        # 3. token_to_words : set in-batch max_word_length\n",
        "        max_word_length = max(torch.sum(bpe_head_masks, dim=1)).item()\n",
        "        # 3. token_to_words : placeholders\n",
        "        head_ids = torch.zeros(batch_size, max_word_length).long()\n",
        "        type_ids = torch.zeros(batch_size, max_word_length).long()\n",
        "        pos_ids = torch.zeros(batch_size, max_word_length + 1).long()\n",
        "        mask_e = torch.zeros(batch_size, max_word_length + 1).long()\n",
        "        # 3. token_to_words : head_ids, type_ids, pos_ids, mask_e, mask_d\n",
        "        for batch_id in range(batch_size):\n",
        "            (\n",
        "                _,\n",
        "                _,\n",
        "                bpe_head_mask,\n",
        "                _,\n",
        "                token_head_ids,\n",
        "                token_type_ids,\n",
        "                token_pos_ids,\n",
        "            ) = batch[batch_id]\n",
        "            # head_id : [1, 3, 5] (prediction candidates)\n",
        "            # token_head_ids : [-1, 3, -1, 3, -1, 0, -1, -1, -1, .-1, ...] (ground truth head ids)\n",
        "            head_id = [i for i, token in enumerate(bpe_head_mask) if token == 1]\n",
        "            word_length = len(head_id)\n",
        "            head_id.extend([0] * (max_word_length - word_length))\n",
        "            head_ids[batch_id] = token_head_ids[head_id]\n",
        "            type_ids[batch_id] = token_type_ids[head_id]\n",
        "            pos_ids[batch_id][0] = torch.tensor(pos_padding_idx)\n",
        "            pos_ids[batch_id][1:] = token_pos_ids[head_id]\n",
        "            pos_ids[batch_id][int(torch.sum(bpe_head_mask)) + 1 :] = torch.tensor(pos_padding_idx)\n",
        "            mask_e[batch_id] = torch.LongTensor([1] * (word_length + 1) + [0] * (max_word_length - word_length))\n",
        "        mask_d = mask_e[:, 1:]\n",
        "        # 4. pack everything\n",
        "        masks = (attention_masks, bpe_head_masks, bpe_tail_masks, mask_e, mask_d)\n",
        "        ids = (head_ids, type_ids, pos_ids)\n",
        "        \"\"\"\n",
        "            self,\n",
        "            input_ids=None,\n",
        "            attention_mask=None,\n",
        "            bpe_head_masks=None,\n",
        "            bpe_tail_masks=None,\n",
        "            mask_e=None,\n",
        "            mask_d=None,\n",
        "            head_labels=None,\n",
        "            type_labels=None,\n",
        "            pos_ids=None,\n",
        "            **kwargs,\n",
        "        \"\"\"\n",
        "        return {\"input_ids\": input_ids, \"attention_masks\": attention_masks, \"bpe_head_masks\":bpe_head_masks, \"max_word_length\":max_word_length, \\\n",
        "                \"bpe_tail_masks\": bpe_tail_masks, \"mask_e\": mask_e, \"mask_d\": mask_d, \"head_labels\" : head_ids, \"type_labels\" : type_ids, \"pos_ids\" : pos_ids}\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "id": "JxykY6euEWeP",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JxykY6euEWeP",
        "outputId": "c531f520-93e6-4916-e8bc-9fff792fe73c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(tensor([    2,  7443,  2259,  4153,  2079,  4646,  2470,  4103,  2170,  3618,\n",
              "          4901,  2116, 28959,  8248,  1670,  2483,  2522,   648,  2483,  2116,\n",
              "          4006,  2125,  3992,  2138,  1170,  4000,  1540,  2069,  4089,  2371,\n",
              "          2062,    18,     3,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "             0,     0,     0,     0,     0,     0,     0,     0]),\n",
              " tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
              "         1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0,\n",
              "         1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " tensor([0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1,\n",
              "         0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
              "         0, 0, 0, 0, 0, 0, 0, 0]),\n",
              " tensor([-100,   15, -100,    4, -100,    4, -100,    5, -100,    6,    7, -100,\n",
              "            8,   13,   10, -100, -100,   13, -100, -100,   12, -100,   13, -100,\n",
              "           14, -100,   15, -100,    0, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100]),\n",
              " tensor([-100,    3, -100,    8, -100,    4, -100,    1, -100,    4,    3, -100,\n",
              "            2,    2,    7, -100, -100,    3, -100, -100,   10, -100,    5, -100,\n",
              "            4, -100,    5, -100,    2, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100]),\n",
              " tensor([-100,   24, -100,   19, -100,   29, -100,   21, -100,   29,   17, -100,\n",
              "           27,   27,   15, -100, -100,   17, -100, -100,   31, -100,   20, -100,\n",
              "           29, -100,   20, -100,   35, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100,\n",
              "         -100, -100, -100, -100, -100, -100, -100, -100]))"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataprocessor = KlueDPProcessor(128, tokenizer)\n",
        "tasks, raw_datasets = load_datasets(dataprocessor, data_args)\n",
        "raw_datasets['validation'][1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "id": "QLUtpezyrTiP",
      "metadata": {
        "id": "QLUtpezyrTiP"
      },
      "outputs": [],
      "source": [
        "\n",
        "class BiAttention(nn.Module):\n",
        "    def __init__(  # type: ignore[no-untyped-def]\n",
        "        self, input_size_encoder: int, input_size_decoder: int, num_labels: int, biaffine: bool = True, **kwargs\n",
        "    ) -> None:\n",
        "        super(BiAttention, self).__init__()\n",
        "        self.input_size_encoder = input_size_encoder\n",
        "        self.input_size_decoder = input_size_decoder\n",
        "        self.num_labels = num_labels\n",
        "        self.biaffine = biaffine\n",
        "\n",
        "        self.W_e = Parameter(torch.Tensor(self.num_labels, self.input_size_encoder))\n",
        "        self.W_d = Parameter(torch.Tensor(self.num_labels, self.input_size_decoder))\n",
        "        self.b = Parameter(torch.Tensor(self.num_labels, 1, 1))\n",
        "        if self.biaffine:\n",
        "            self.U = Parameter(torch.Tensor(self.num_labels, self.input_size_decoder, self.input_size_encoder))\n",
        "        else:\n",
        "            self.register_parameter(\"U\", None)\n",
        "\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self) -> None:\n",
        "        nn.init.xavier_uniform_(self.W_e)\n",
        "        nn.init.xavier_uniform_(self.W_d)\n",
        "        nn.init.constant_(self.b, 0.0)\n",
        "        if self.biaffine:\n",
        "            nn.init.xavier_uniform_(self.U)\n",
        "\n",
        "    def forward(\n",
        "        self,\n",
        "        input_d: torch.Tensor,\n",
        "        input_e: torch.Tensor,\n",
        "        mask_d: Optional[torch.Tensor] = None,\n",
        "        mask_e: Optional[torch.Tensor] = None,\n",
        "    ) -> torch.Tensor:\n",
        "        assert input_d.size(0) == input_e.size(0)\n",
        "        batch, length_decoder, _ = input_d.size()\n",
        "        _, length_encoder, _ = input_e.size()\n",
        "\n",
        "        out_d = torch.matmul(self.W_d, input_d.transpose(1, 2)).unsqueeze(3)\n",
        "        out_e = torch.matmul(self.W_e, input_e.transpose(1, 2)).unsqueeze(2)\n",
        "\n",
        "        if self.biaffine:\n",
        "            output = torch.matmul(input_d.unsqueeze(1), self.U)\n",
        "            output = torch.matmul(output, input_e.unsqueeze(1).transpose(2, 3))\n",
        "            output = output + out_d + out_e + self.b\n",
        "        else:\n",
        "            output = out_d + out_d + self.b\n",
        "\n",
        "        if mask_d is not None:\n",
        "            output = output * mask_d.unsqueeze(1).unsqueeze(3) * mask_e.unsqueeze(1).unsqueeze(2)\n",
        "\n",
        "        return output\n",
        "\n",
        "\n",
        "class BiLinear(nn.Module):\n",
        "    def __init__(self, left_features: int, right_features: int, out_features: int):\n",
        "        super(BiLinear, self).__init__()\n",
        "        self.left_features = left_features\n",
        "        self.right_features = right_features\n",
        "        self.out_features = out_features\n",
        "\n",
        "        self.U = Parameter(torch.Tensor(self.out_features, self.left_features, self.right_features))\n",
        "        self.W_l = Parameter(torch.Tensor(self.out_features, self.left_features))\n",
        "        self.W_r = Parameter(torch.Tensor(self.out_features, self.left_features))\n",
        "        self.bias = Parameter(torch.Tensor(out_features))\n",
        "\n",
        "        self.reset_parameters()\n",
        "\n",
        "    def reset_parameters(self) -> None:\n",
        "        nn.init.xavier_uniform_(self.W_l)\n",
        "        nn.init.xavier_uniform_(self.W_r)\n",
        "        nn.init.constant_(self.bias, 0.0)\n",
        "        nn.init.xavier_uniform_(self.U)\n",
        "\n",
        "    def forward(self, input_left: torch.Tensor, input_right: torch.Tensor) -> torch.Tensor:\n",
        "        left_size = input_left.size()\n",
        "        right_size = input_right.size()\n",
        "        assert left_size[:-1] == right_size[:-1], \"batch size of left and right inputs mis-match: (%s, %s)\" % (\n",
        "            left_size[:-1],\n",
        "            right_size[:-1],\n",
        "        )\n",
        "        batch = int(np.prod(left_size[:-1]))\n",
        "\n",
        "        input_left = input_left.contiguous().view(batch, self.left_features)\n",
        "        input_right = input_right.contiguous().view(batch, self.right_features)\n",
        "\n",
        "        output = F.bilinear(input_left, input_right, self.U, self.bias)\n",
        "        output = output + F.linear(input_left, self.W_l, None) + F.linear(input_right, self.W_r, None)\n",
        "        return output.view(left_size[:-1] + (self.out_features,))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "id": "NCyA_WVu-b5J",
      "metadata": {
        "id": "NCyA_WVu-b5J"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "class TokenClassificationHead(nn.Module):\n",
        "    def __init__(self, hidden_size, task, dropout_p=0.1):\n",
        "        super().__init__()\n",
        "        self.dropout = nn.Dropout2d(p=0.33)\n",
        "        self.num_labels = task.num_labels\n",
        "        self.hidden_size = hidden_size\n",
        "        self.task_id = task.id \n",
        "        self.n_pos_labels = len(get_pos_labels())\n",
        "        self.n_dp_labels = len(get_dep_labels())\n",
        "\n",
        "        self.pos_dim = 128\n",
        "        self.arc_space = 512\n",
        "        self.type_space = 256\n",
        "        self.pos_embedding = nn.Embedding(self.n_pos_labels + 1, self.pos_dim)\n",
        "\n",
        "        enc_dim = self.hidden_size * 2\n",
        "        if self.pos_embedding is not None:\n",
        "            enc_dim += self.pos_dim\n",
        "\n",
        "\n",
        "\n",
        "        # Bidirectional LSTM encoder\n",
        "        self.encoder = nn.LSTM(enc_dim, self.hidden_size, batch_first=True,dropout = 0.33, bidirectional=True)\n",
        "\n",
        "        # Unidirectional LSTM decoder\n",
        "        self.decoder = nn.LSTM(self.hidden_size , self.hidden_size, batch_first=True)\n",
        "\n",
        "        self.src_dense = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
        "        self.hx_dense = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
        "\n",
        "        self.classifier = nn.Linear(self.hidden_size, self.num_labels)\n",
        "\n",
        "        if str(self.task_id) == \"0\": \n",
        "\n",
        "          self.arc_c = nn.Linear(self.hidden_size * 2, self.arc_space)\n",
        "          self.arc_h = nn.Linear(self.hidden_size, self.arc_space)\n",
        "          self.attention = BiAttention(self.arc_space, self.arc_space, self.num_labels)\n",
        "\n",
        "        else:\n",
        "\n",
        "          self.type_c = nn.Linear(self.hidden_size * 2, self.type_space)\n",
        "          self.type_h = nn.Linear(self.hidden_size, self.type_space)\n",
        "          self.bilinear = BiLinear(self.type_space, self.type_space, self.num_labels)\n",
        "\n",
        "\n",
        "\n",
        "        self._init_weights()\n",
        "\n",
        "    def _init_weights(self):\n",
        "        self.classifier.weight.data.normal_(mean=0.0, std=0.02)\n",
        "        if self.classifier.bias is not None:\n",
        "            self.classifier.bias.data.zero_()\n",
        "\n",
        "    def forward(\n",
        "        self, sequence_output, labels=None, pos_ids = None, max_word_length=None,  head_masks=None, tail_masks=None,mask_e=None, mask_d=None, task = None, **kwargs\n",
        "    ):\n",
        "\n",
        "        outputs = sequence_output\n",
        "        outputs, sent_len = self.resize_outputs(outputs, head_masks, tail_masks, max_word_length)\n",
        "\n",
        "\n",
        "        if self.pos_embedding is not None:\n",
        "            pos_outputs = self.pos_embedding(pos_ids)\n",
        "            pos_outputs = self.dropout(pos_outputs)\n",
        "            outputs = torch.cat([outputs, pos_outputs], dim=2)\n",
        "\n",
        "        # encoder\n",
        "        packed_outputs = pack_padded_sequence(outputs, sent_len, batch_first=True, enforce_sorted=False)\n",
        "        encoder_outputs, hn = self.encoder(packed_outputs)\n",
        "        encoder_outputs, outputs_len = pad_packed_sequence(encoder_outputs, batch_first=True)\n",
        "        encoder_outputs = self.dropout(encoder_outputs.transpose(1, 2)).transpose(1, 2)  # apply dropout for last layer\n",
        "        hn = self._transform_decoder_init_state(hn)\n",
        "\n",
        "        # decoder\n",
        "        src_encoding = F.elu(self.src_dense(encoder_outputs[:, 1:]))\n",
        "        sent_len = [i - 1 for i in sent_len]\n",
        "        packed_outputs = pack_padded_sequence(src_encoding, sent_len, batch_first=True, enforce_sorted=False)\n",
        "        decoder_outputs, _ = self.decoder(packed_outputs, hn)\n",
        "        decoder_outputs, outputs_len = pad_packed_sequence(decoder_outputs, batch_first=True)\n",
        "        decoder_outputs = self.dropout(decoder_outputs.transpose(1, 2)).transpose(1, 2)  # apply dropout for last layer\n",
        "\n",
        "        #TODO : Finish the attention mechansim part\n",
        "        # compute output for arc and type\n",
        "        #if str(task.id) == '0':\n",
        "\n",
        "        #  arc_c = F.elu(self.arc_c(encoder_outputs))\n",
        "        #  arc_h = F.elu(self.arc_h(decoder_outputs))\n",
        "        #  logits = self.attention(arc_h, arc_c, mask_d=mask_d, mask_e=mask_e).squeeze(dim=1)\n",
        "          #print(logits.shape) (batch, num_label, 20, 21)\n",
        "        #else:\n",
        "\n",
        "        #  type_c = F.elu(self.type_c(encoder_outputs))\n",
        "        #  type_h = F.elu(self.type_h(decoder_outputs))\n",
        "        #  logits = self.bilinear(type_h, type_c)\n",
        "          #print(logits.shape)\n",
        "\n",
        "\n",
        "        # Pass the decoder output through the classifier\n",
        "        logits = self.classifier(decoder_outputs)\n",
        "        \n",
        "        #additional masking\n",
        "        if str(task.id) == '0':\n",
        "          minus_inf = -1e8\n",
        "\n",
        "          # Calculate the required padding size along the last dimension\n",
        "          padding_size = task.num_labels - mask_e.size(-1)\n",
        "\n",
        "          # Create a tensor of zeros with the same size as mask_e along the first dimension and padding_size along the last dimension\n",
        "          zeros_padding = torch.zeros(mask_e.size(0), padding_size).to(outputs.device)\n",
        "\n",
        "          # Concatenate mask_e with the zeros_padding tensor along the last dimension\n",
        "          mask_e_padded = torch.cat((mask_e, zeros_padding), dim=-1)\n",
        "\n",
        "          minus_mask_e = (1 - mask_e_padded) * minus_inf\n",
        "          logits = logits + minus_mask_e.unsqueeze(1) \n",
        "\n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "            loss_fct = torch.nn.CrossEntropyLoss()\n",
        "\n",
        "            labels = labels.long()\n",
        "\n",
        "            # Only keep active parts of the loss\n",
        "            if mask_d is not None:\n",
        "                active_loss = mask_d.view(-1) == 1\n",
        "\n",
        "                active_logits = logits.view(-1, self.num_labels)\n",
        "                active_labels = torch.where(\n",
        "                    active_loss,\n",
        "                    labels.view(-1),\n",
        "                    torch.tensor(loss_fct.ignore_index).type_as(labels),\n",
        "                )\n",
        "    \n",
        "                loss = loss_fct(active_logits, active_labels)\n",
        "            else:\n",
        "                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n",
        "\n",
        "        return logits, loss\n",
        "\n",
        "    def resize_outputs(\n",
        "        self, outputs: torch.Tensor, bpe_head_mask: torch.Tensor, bpe_tail_mask: torch.Tensor, max_word_length: int\n",
        "    ) -> Tuple[torch.Tensor, List]:\n",
        "        \"\"\"Resize output of pre-trained transformers (bsz, max_token_length, hidden_dim) to word-level outputs (bsz, max_word_length, hidden_dim*2). \"\"\"\n",
        "        batch_size, input_size, hidden_size = outputs.size()\n",
        "        word_outputs = torch.zeros(batch_size, max_word_length + 1, hidden_size * 2).to(outputs.device)\n",
        "        sent_len = list()\n",
        "\n",
        "        for batch_id in range(batch_size):\n",
        "            head_ids = [i for i, token in enumerate(bpe_head_mask[batch_id]) if token == 1]\n",
        "            tail_ids = [i for i, token in enumerate(bpe_tail_mask[batch_id]) if token == 1]\n",
        "            assert len(head_ids) == len(tail_ids)\n",
        "\n",
        "            word_outputs[batch_id][0] = torch.cat(\n",
        "                (outputs[batch_id][0], outputs[batch_id][0])\n",
        "            )  # replace root with [CLS]\n",
        "            for i, (head, tail) in enumerate(zip(head_ids, tail_ids)):\n",
        "                word_outputs[batch_id][i + 1] = torch.cat((outputs[batch_id][head], outputs[batch_id][tail]))\n",
        "            sent_len.append(i + 2)\n",
        "\n",
        "        return word_outputs, sent_len\n",
        "\n",
        "    def _transform_decoder_init_state(self, hn: torch.Tensor) -> torch.Tensor:\n",
        "        hn, cn = hn\n",
        "        cn = cn[-2:]  # take the last layer\n",
        "        _, batch_size, hidden_size = cn.size()\n",
        "        cn = cn.transpose(0, 1).contiguous()\n",
        "        cn = cn.view(batch_size, 1, 2 * hidden_size).transpose(0, 1)\n",
        "        cn = self.hx_dense(cn)\n",
        "        if self.decoder.num_layers > 1:\n",
        "            cn = torch.cat(\n",
        "                [\n",
        "                    cn,\n",
        "                    torch.autograd.Variable(cn.data.new(self.decoder.num_layers - 1, batch_size, hidden_size).zero_()),\n",
        "                ],\n",
        "                dim=0,\n",
        "            )\n",
        "        hn = torch.tanh(cn)\n",
        "        hn = (hn, cn)\n",
        "        return hn"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "id": "nHY3Z03fa7Ak",
      "metadata": {
        "id": "nHY3Z03fa7Ak"
      },
      "outputs": [],
      "source": [
        "\n",
        "class SequenceClassificationHead(nn.Module):\n",
        "    def __init__(self, hidden_size, num_labels, dropout_p=0.1):\n",
        "        super().__init__()\n",
        "        self.num_labels = num_labels\n",
        "        self.dropout = nn.Dropout(dropout_p)\n",
        "        self.classifier = nn.Linear(hidden_size, num_labels)\n",
        "\n",
        "        self._init_weights()\n",
        "\n",
        "    def forward(self, sequence_output, pooled_output, labels=None, **kwargs):\n",
        "        pooled_output = self.dropout(pooled_output)\n",
        "        logits = self.classifier(pooled_output)\n",
        "\n",
        "        loss = None\n",
        "        if labels is not None:\n",
        "            if labels.dim() != 1:\n",
        "                # Remove padding\n",
        "                labels = labels[:, 0]\n",
        "\n",
        "            loss_fct = nn.CrossEntropyLoss()\n",
        "            loss = loss_fct(\n",
        "                logits.view(-1, self.num_labels), labels.long().view(-1)\n",
        "            )\n",
        "\n",
        "        return logits, loss\n",
        "\n",
        "    def _init_weights(self):\n",
        "        self.classifier.weight.data.normal_(mean=0.0, std=0.02)\n",
        "        if self.classifier.bias is not None:\n",
        "            self.classifier.bias.data.zero_()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "id": "Rh1iddokCMq3",
      "metadata": {
        "id": "Rh1iddokCMq3"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "id": "TjPplSL8a7Lu",
      "metadata": {
        "id": "TjPplSL8a7Lu"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "id": "LfPg6Z6ezs6G",
      "metadata": {
        "id": "LfPg6Z6ezs6G"
      },
      "outputs": [],
      "source": [
        "\n",
        "class MultiTaskModel(nn.Module):\n",
        "    def __init__(self, encoder_name_or_path, tasks: List):\n",
        "        super().__init__()\n",
        "        self.encoder = AutoModel.from_pretrained(encoder_name_or_path, config = AutoConfig.from_pretrained(encoder_name_or_path))\n",
        "        self.tasks = tasks\n",
        "        self.output_heads = nn.ModuleDict()\n",
        "        for task in self.tasks :\n",
        "            decoder = self._create_output_head(self.encoder.config.hidden_size, task)\n",
        "            # ModuleDict requires keys to be strings\n",
        "            self.output_heads[str(task.id)] = decoder\n",
        "\n",
        "    @staticmethod\n",
        "    def _create_output_head(encoder_hidden_size: int, task):\n",
        "        if task.type == \"seq_classification\":\n",
        "            return SequenceClassificationHead(encoder_hidden_size, task)\n",
        "        elif task.type == \"token_classification\":\n",
        "            return TokenClassificationHead(encoder_hidden_size, task)\n",
        "        else:\n",
        "            raise NotImplementedError()\n",
        "\n",
        "    def forward(\n",
        "            self,\n",
        "            input_ids=None,\n",
        "            attention_mask=None,\n",
        "            bpe_head_masks=None,\n",
        "            bpe_tail_masks=None,\n",
        "            max_word_length=None,\n",
        "            mask_e=None,\n",
        "            mask_d=None,\n",
        "            head_labels=None,\n",
        "            type_labels=None,\n",
        "            pos_ids=None,\n",
        "            **kwargs,\n",
        "        ):\n",
        "            \"\"\"\n",
        "                self,\n",
        "            input_ids=None,\n",
        "            attention_mask=None,\n",
        "            token_type_ids=None,\n",
        "            position_ids=None,\n",
        "            head_mask=None,\n",
        "            inputs_embeds=None,\n",
        "            labels=None,\n",
        "            **kwargs,\n",
        "            \"\"\"\n",
        "            \n",
        "            outputs = self.encoder(\n",
        "                input_ids=input_ids,\n",
        "                attention_mask=attention_mask\n",
        "            )\n",
        "            \n",
        "            sequence_output, pooled_output = outputs[:2]\n",
        "\n",
        "            unique_task_ids_list = self.tasks \n",
        "\n",
        "            loss_list = []\n",
        "            logits_list = []\n",
        "\n",
        "            logits = None\n",
        " \n",
        "            gt = [ head_labels, type_labels] #type_ids is dep_ids\n",
        "            for task in self.tasks :\n",
        "                labels = gt[task.id]\n",
        "                logits, task_loss = self.output_heads[str(task.id)].forward(\n",
        "                    sequence_output,\n",
        "                    labels=labels,\n",
        "                    max_word_length= max_word_length,\n",
        "                    pos_ids = pos_ids,\n",
        "                    head_masks=bpe_head_masks,\n",
        "                    tail_masks=bpe_tail_masks,\n",
        "                    mask_e = mask_e, \n",
        "                    mask_d = mask_d, \n",
        "                    task = task\n",
        "                )\n",
        "                logits_list.append(logits)\n",
        "                if labels is not None:\n",
        "                    loss_list.append(task_loss)\n",
        "\n",
        "            # logits are only used for eval. and in case of eval the batch is not multi task\n",
        "            # For training only the loss is used\n",
        "\n",
        "            if loss_list:\n",
        "                loss = torch.stack(loss_list)\n",
        "            return {\"loss\":loss.mean(), \"logits\":logits_list}\n",
        "            #outputs = (logits, outputs[2:])\n",
        "            #return TokenClassifierOutput(loss=loss, logits=logits)\n",
        "            #return outputs # (loss, outputs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "id": "D1Rvl7Y50AZY",
      "metadata": {
        "id": "D1Rvl7Y50AZY"
      },
      "outputs": [],
      "source": [
        "from seqeval.metrics import f1_score\n",
        "from seqeval.metrics import accuracy_score\n",
        "from seqeval.metrics import classification_report"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "id": "JZgayV9F0EKh",
      "metadata": {
        "id": "JZgayV9F0EKh"
      },
      "outputs": [],
      "source": [
        "def compute_metrics(p: EvalPrediction): #EvalPrediction argument have predictions and label_ids as parameters \n",
        "    preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions\n",
        "\n",
        "\n",
        "    #if preds.ndim == 2:\n",
        "    # Token classification\n",
        "      #preds = np.argmax(preds, axis=1)\n",
        "      #logger.warning({\"accuracy\": (preds == p.label_ids).astype(np.float32).mean().item()})\n",
        "      #return {\"accuracy\": (preds == p.label_ids).astype(np.float32).mean().item()}\n",
        "  \n",
        "    # Sequence classification\n",
        "    result_dict = {}\n",
        "\n",
        "    for i in range(len(p.predictions)):\n",
        "      preds = p.predictions[i]\n",
        "      #https://github.com/chakki-works/seqeval/blob/cd01b5210eaa65e691c22320aba56f2be9e9fc43/seqeval/metrics/v1.py#L136\n",
        "      metric = load_metric(\"seqeval\")\n",
        "\n",
        "      predictions = np.argmax(preds, axis=2)\n",
        "\n",
        "      true_predictions = [\n",
        "          [str(p) for (p, l) in zip(prediction, label) if l != -100]\n",
        "          for prediction, label in zip(predictions, p.label_ids[i])\n",
        "      ]\n",
        "      true_labels = [\n",
        "          [str(l) for (p, l) in zip(prediction, label) if l != -100]\n",
        "          for prediction, label in zip(predictions, p.label_ids[i])\n",
        "      ]\n",
        "\n",
        "      #print({\"accuracy\": np.array((true_predictions == true_labels)).astype(np.float32).mean().item()})\n",
        "      # Remove ignored index (special tokens)\n",
        "      results = metric.compute(\n",
        "          predictions=true_predictions, references=true_labels, zero_division = 0\n",
        "      )    \n",
        "      #print(classification_report(true_labels, true_predictions))\n",
        "\n",
        "\n",
        "      result_dict[f\"f1_{i}\"] = f1_score(true_labels, true_predictions)\n",
        "      result_dict[f\"accuracy_{i}\"] = accuracy_score(true_labels, true_predictions)\n",
        "    return result_dict\n",
        "\n",
        " "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "id": "gomPRjwurShy",
      "metadata": {
        "id": "gomPRjwurShy"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "id": "bGPI2pRrx-m7",
      "metadata": {
        "id": "bGPI2pRrx-m7"
      },
      "outputs": [],
      "source": [
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "id": "SGZVFfIT0L6M",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "SGZVFfIT0L6M",
        "outputId": "afdaa040-f532-4b0e-a30d-cbeb2e389c70"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:__main__:Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
            "[WARNING|modeling_utils.py:3180] 2023-05-04 03:29:08,166 >> Some weights of the model checkpoint at klue/bert-base were not used when initializing BertModel: ['cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "WARNING:__main__:Training\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='25000' max='25000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [25000/25000 3:05:02, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1 0</th>\n",
              "      <th>Accuracy 0</th>\n",
              "      <th>F1 1</th>\n",
              "      <th>Accuracy 1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.792700</td>\n",
              "      <td>0.494106</td>\n",
              "      <td>0.316858</td>\n",
              "      <td>0.697324</td>\n",
              "      <td>0.687154</td>\n",
              "      <td>0.962616</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.409100</td>\n",
              "      <td>0.345901</td>\n",
              "      <td>0.588249</td>\n",
              "      <td>0.808099</td>\n",
              "      <td>0.821979</td>\n",
              "      <td>0.972262</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.325800</td>\n",
              "      <td>0.290099</td>\n",
              "      <td>0.683923</td>\n",
              "      <td>0.853396</td>\n",
              "      <td>0.840858</td>\n",
              "      <td>0.974529</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.247200</td>\n",
              "      <td>0.268021</td>\n",
              "      <td>0.715550</td>\n",
              "      <td>0.867710</td>\n",
              "      <td>0.844930</td>\n",
              "      <td>0.974040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.215000</td>\n",
              "      <td>0.256570</td>\n",
              "      <td>0.747130</td>\n",
              "      <td>0.879267</td>\n",
              "      <td>0.855776</td>\n",
              "      <td>0.975773</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.181100</td>\n",
              "      <td>0.250955</td>\n",
              "      <td>0.769758</td>\n",
              "      <td>0.889403</td>\n",
              "      <td>0.852934</td>\n",
              "      <td>0.975951</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.159600</td>\n",
              "      <td>0.251962</td>\n",
              "      <td>0.780886</td>\n",
              "      <td>0.893803</td>\n",
              "      <td>0.861837</td>\n",
              "      <td>0.975685</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.135400</td>\n",
              "      <td>0.252610</td>\n",
              "      <td>0.786688</td>\n",
              "      <td>0.897004</td>\n",
              "      <td>0.854595</td>\n",
              "      <td>0.975284</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.121300</td>\n",
              "      <td>0.252773</td>\n",
              "      <td>0.794869</td>\n",
              "      <td>0.902649</td>\n",
              "      <td>0.850037</td>\n",
              "      <td>0.975196</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.109700</td>\n",
              "      <td>0.249646</td>\n",
              "      <td>0.805602</td>\n",
              "      <td>0.906561</td>\n",
              "      <td>0.858824</td>\n",
              "      <td>0.975018</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.099000</td>\n",
              "      <td>0.253493</td>\n",
              "      <td>0.808812</td>\n",
              "      <td>0.909228</td>\n",
              "      <td>0.860147</td>\n",
              "      <td>0.975640</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.086800</td>\n",
              "      <td>0.259210</td>\n",
              "      <td>0.815042</td>\n",
              "      <td>0.911629</td>\n",
              "      <td>0.855051</td>\n",
              "      <td>0.975418</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.082300</td>\n",
              "      <td>0.257953</td>\n",
              "      <td>0.818702</td>\n",
              "      <td>0.913807</td>\n",
              "      <td>0.849047</td>\n",
              "      <td>0.975373</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.077200</td>\n",
              "      <td>0.262589</td>\n",
              "      <td>0.821226</td>\n",
              "      <td>0.914874</td>\n",
              "      <td>0.861805</td>\n",
              "      <td>0.976174</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.070100</td>\n",
              "      <td>0.265047</td>\n",
              "      <td>0.819998</td>\n",
              "      <td>0.914829</td>\n",
              "      <td>0.860385</td>\n",
              "      <td>0.975551</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.067400</td>\n",
              "      <td>0.263479</td>\n",
              "      <td>0.823590</td>\n",
              "      <td>0.916785</td>\n",
              "      <td>0.863961</td>\n",
              "      <td>0.975862</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.064300</td>\n",
              "      <td>0.267970</td>\n",
              "      <td>0.824173</td>\n",
              "      <td>0.916874</td>\n",
              "      <td>0.857982</td>\n",
              "      <td>0.975773</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.059800</td>\n",
              "      <td>0.269450</td>\n",
              "      <td>0.826644</td>\n",
              "      <td>0.917541</td>\n",
              "      <td>0.863558</td>\n",
              "      <td>0.976307</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.058400</td>\n",
              "      <td>0.270192</td>\n",
              "      <td>0.827367</td>\n",
              "      <td>0.917719</td>\n",
              "      <td>0.866437</td>\n",
              "      <td>0.976218</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.057400</td>\n",
              "      <td>0.270498</td>\n",
              "      <td>0.827188</td>\n",
              "      <td>0.917319</td>\n",
              "      <td>0.862426</td>\n",
              "      <td>0.976351</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "***** train metrics *****\n",
            "  epoch                    =       20.0\n",
            "  total_flos               =        0GF\n",
            "  train_loss               =     0.1898\n",
            "  train_runtime            = 3:05:03.39\n",
            "  train_samples            =      10000\n",
            "  train_samples_per_second =     18.013\n",
            "  train_steps_per_second   =      2.252\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='500' max='250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [250/250 01:38]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "***** eval metrics *****\n",
            "  epoch                   =       20.0\n",
            "  eval_accuracy_0         =     0.9173\n",
            "  eval_accuracy_1         =     0.9764\n",
            "  eval_f1_0               =     0.8272\n",
            "  eval_f1_1               =     0.8624\n",
            "  eval_loss               =     0.2705\n",
            "  eval_runtime            = 0:00:50.81\n",
            "  eval_samples            =       2000\n",
            "  eval_samples_per_second =      39.36\n",
            "  eval_steps_per_second   =       4.92\n",
            "***** eval metrics *****\n",
            "  epoch                   =       20.0\n",
            "  eval_accuracy_0         =     0.9173\n",
            "  eval_accuracy_1         =     0.9764\n",
            "  eval_f1_0               =     0.8272\n",
            "  eval_f1_1               =     0.8624\n",
            "  eval_loss               =     0.2705\n",
            "  eval_runtime            = 0:00:50.65\n",
            "  eval_samples            =       2000\n",
            "  eval_samples_per_second =     39.485\n",
            "  eval_steps_per_second   =      4.936\n"
          ]
        }
      ],
      "source": [
        "def main(model_args, data_args, training_args):\n",
        "\n",
        "    # Setup logging\n",
        "    logging.basicConfig(\n",
        "        format=\"%(asctime)s - %(levelname)s - %(name)s - %(message)s\",\n",
        "        datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
        "        handlers=[logging.StreamHandler(sys.stdout)],\n",
        "    )\n",
        "    \n",
        "    log_level = training_args.get_process_log_level()\n",
        "    logger.setLevel(logging.WARNING)\n",
        "    datasets.utils.logging.set_verbosity(log_level)\n",
        "    transformers.utils.logging.set_verbosity(log_level)\n",
        "    transformers.utils.logging.enable_default_handler()\n",
        "    transformers.utils.logging.enable_explicit_format()\n",
        "\n",
        "    # Log on each process the small summary:\n",
        "    logger.warning(\n",
        "        f\"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}\"\n",
        "        + f\"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}\"\n",
        "    )\n",
        "    logger.info(f\"Training/evaluation parameters {training_args}\")\n",
        "\n",
        "    # Detecting last checkpoint.\n",
        "    last_checkpoint = None\n",
        "    if (\n",
        "        os.path.isdir(training_args.output_dir)\n",
        "        and training_args.do_train\n",
        "        and not training_args.overwrite_output_dir\n",
        "    ):\n",
        "        last_checkpoint = get_last_checkpoint(training_args.output_dir)\n",
        "        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:\n",
        "            raise ValueError(\n",
        "                f\"Output directory ({training_args.output_dir}) already exists and is not empty. \"\n",
        "                \"Use --overwrite_output_dir to overcome.\"\n",
        "            )\n",
        "        elif (\n",
        "            last_checkpoint is not None and training_args.resume_from_checkpoint is None\n",
        "        ):\n",
        "            logger.info(\n",
        "                f\"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change \"\n",
        "                \"the `--output_dir` or add `--overwrite_output_dir` to train from scratch.\"\n",
        "            )\n",
        "\n",
        "    set_seed(training_args.seed)\n",
        "\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\n",
        "        model_args.encoder_name_or_path,\n",
        "        cache_dir=model_args.cache_dir,\n",
        "        use_fast=model_args.use_fast_tokenizer,\n",
        "        revision=model_args.model_revision,\n",
        "        use_auth_token=True if model_args.use_auth_token else None,\n",
        "    )\n",
        "    dataprocessor = KlueDPProcessor(128, tokenizer)\n",
        "\n",
        "    tasks, raw_datasets = load_datasets(dataprocessor, data_args)\n",
        "\n",
        "    model = MultiTaskModel(model_args.encoder_name_or_path, tasks)\n",
        "\n",
        "    if training_args.do_train:\n",
        "        if \"train\" not in raw_datasets:\n",
        "            raise ValueError(\"--do_train requires a train dataset\")\n",
        "        train_dataset = raw_datasets[\"train\"]\n",
        "        if data_args.max_train_samples is not None:\n",
        "            train_dataset = train_dataset.select(range(data_args.max_train_samples))\n",
        "\n",
        "    if training_args.do_eval:\n",
        "        \n",
        "        if (\n",
        "            \"validation\" not in raw_datasets\n",
        "            and \"validation_matched\" not in raw_datasets\n",
        "        ):\n",
        "            raise ValueError(\"--do_eval requires a validation dataset\")\n",
        "        eval_dataset = raw_datasets[\"validation\"]\n",
        "        \"\"\"\n",
        "        if data_args.max_eval_samples is not None:\n",
        "            new_ds = []\n",
        "            for ds in eval_datasets:\n",
        "                new_ds.append(ds.select(range(data_args.max_eval_samples)))\n",
        "\n",
        "            eval_datasets = new_ds\n",
        "        \"\"\"\n",
        "    # Log a few random samples from the training set:\n",
        "    if training_args.do_train:\n",
        "        for index in random.sample(range(len(train_dataset)), 3):\n",
        "            logger.info(f\"Sample {index} of the training set: {train_dataset[index]}.\")\n",
        "\n",
        "    # Log a few random samples from the training set:\n",
        "    if training_args.do_eval:\n",
        "        for index in random.sample(range(len(eval_dataset)), 3):\n",
        "            logger.info(f\"Sample {index} of the eval set: {eval_dataset[index]}.\")\n",
        "\n",
        "\n",
        "    data_collator = DataCollatorForTokenClassification(\n",
        "        tokenizer, pad_to_multiple_of=8 if training_args.fp16 else None\n",
        "    )\n",
        "\n",
        "    # Initialize our Trainer\n",
        "    trainer = Trainer(\n",
        "        model=model,\n",
        "        args=training_args,\n",
        "        train_dataset=train_dataset if training_args.do_train else None,\n",
        "        eval_dataset=eval_dataset if training_args.do_eval else None,\n",
        "        compute_metrics=compute_metrics,\n",
        "        tokenizer=tokenizer,\n",
        "        data_collator=dataprocessor.collate_fn,\n",
        "    )\n",
        "    logger.warning(\"Training\")\n",
        "    # Training\n",
        "    if training_args.do_train:\n",
        "        checkpoint = None\n",
        "        if training_args.resume_from_checkpoint is not None:\n",
        "            checkpoint = training_args.resume_from_checkpoint\n",
        "        elif last_checkpoint is not None:\n",
        "            checkpoint = last_checkpoint\n",
        "        #train_result = trainer.train(resume_from_checkpoint=checkpoint)\n",
        "        train_result = trainer.train()\n",
        "        metrics = train_result.metrics\n",
        "        max_train_samples = (\n",
        "            data_args.max_train_samples\n",
        "            if data_args.max_train_samples is not None\n",
        "            else len(train_dataset)\n",
        "        )\n",
        "        metrics[\"train_samples\"] = min(max_train_samples, len(train_dataset))\n",
        "\n",
        "        trainer.save_model()  # Saves the tokenizer too for easy upload\n",
        "\n",
        "        trainer.log_metrics(\"train\", metrics)\n",
        "        trainer.save_metrics(\"train\", metrics)\n",
        "        trainer.save_state()\n",
        "\n",
        "    # Evaluation\n",
        "    if training_args.do_eval:\n",
        "        evals = [eval_dataset, eval_dataset]\n",
        "        for eval, task in zip(evals, tasks):\n",
        "\n",
        "\n",
        "            metrics = trainer.evaluate(eval_dataset=eval)\n",
        "            max_eval_samples = (\n",
        "                data_args.max_eval_samples\n",
        "                if data_args.max_eval_samples is not None\n",
        "                else len(eval)\n",
        "            )\n",
        "            metrics[\"eval_samples\"] = min(max_eval_samples, len(eval))\n",
        "\n",
        "            trainer.log_metrics(\"eval\", metrics)\n",
        "            trainer.save_metrics(\"eval\", metrics)\n",
        "            \n",
        "if __name__ == \"__main__\":\n",
        "    model_args = ModelArguments(encoder_name_or_path=\"klue/bert-base\")\n",
        "    training_args = TrainingArguments(\n",
        "        \"test-dp\",\n",
        "        evaluation_strategy=\"epoch\",\n",
        "        save_strategy=\"epoch\",\n",
        "        do_train = True,\n",
        "        do_eval = True,\n",
        "        per_device_train_batch_size = 8, \n",
        "        per_device_eval_batch_size = 8, \n",
        "        learning_rate=3e-5,\n",
        "        num_train_epochs=20,\n",
        "        overwrite_output_dir=True,\n",
        "        logging_dir = './log'\n",
        "    )\n",
        "    data_args = DataTrainingArguments(train_file = \"/content/klue-dp-v1.1_train.tsv\", validation_file = \"/content/klue-dp-v1.1_dev.tsv\", max_seq_length = 128)\n",
        "    main(model_args, data_args, training_args) #validation loss is greater than Training loss because of the drop out "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "RyM7n6G30LxC",
      "metadata": {
        "id": "RyM7n6G30LxC"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "--qPt50e0Lmg",
      "metadata": {
        "id": "--qPt50e0Lmg"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dUyQv7ciz-If",
      "metadata": {
        "id": "dUyQv7ciz-If"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "Cq9vkoyDXTvw",
      "metadata": {
        "id": "Cq9vkoyDXTvw"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "011071963b2d4673bcc64a1bb1563946": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b464d83af294cfd975133ccfafafb68",
            "placeholder": "​",
            "style": "IPY_MODEL_b34136ec3fba4f539478a9bddb035bde",
            "value": " 495k/495k [00:00&lt;00:00, 2.01MB/s]"
          }
        },
        "0df8bf6a7b21437593611d71e0b070eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e47c0c2f4d1e4605907c5828e0744c5e",
            "placeholder": "​",
            "style": "IPY_MODEL_f9f64e83e6024bf38ee31dfaf0aa8f64",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "0fca6111c86c439d9d1956c66c9d8bb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_150121e1ef8d4b62a698c857552b9c4d",
            "placeholder": "​",
            "style": "IPY_MODEL_eb27f6e1353c48318d075be6812799f3",
            "value": " 425/425 [00:00&lt;00:00, 31.9kB/s]"
          }
        },
        "1155b6a9e1b042d8b91d11b34f649954": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "13f4a07998da4904a092831457750a5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7dbf3262548c4087801c93f8a7df5e09",
            "max": 125,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_48faa8b6ebaf4a38beee79d8f8561297",
            "value": 125
          }
        },
        "150121e1ef8d4b62a698c857552b9c4d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b464d83af294cfd975133ccfafafb68": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1bdccc0213cc453bb20e5e4abfd32e35": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "219670845a3a42149f8d534a9d91a4ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "310adfbc64b644c8b454a5d10cd30c39": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "315211d6895c4d198b23f8ccf873b808": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "318a47c48d7149cbb5aaae5328f3a3c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0df8bf6a7b21437593611d71e0b070eb",
              "IPY_MODEL_13f4a07998da4904a092831457750a5d",
              "IPY_MODEL_3e469d02ea9f4c37ae0b4da4c7aeab8e"
            ],
            "layout": "IPY_MODEL_e7219bb4c46a4cb4b70ea67a4c6e3107"
          }
        },
        "37308a8e76214ccc9f2cbef04caa38eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_50d082a0c56a434da92d46b06faf4183",
            "placeholder": "​",
            "style": "IPY_MODEL_f2e571828f4945bf9285bf4de06cb886",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "3e469d02ea9f4c37ae0b4da4c7aeab8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc1467abf51245b7a8b9f1745365893d",
            "placeholder": "​",
            "style": "IPY_MODEL_1155b6a9e1b042d8b91d11b34f649954",
            "value": " 125/125 [00:00&lt;00:00, 5.71kB/s]"
          }
        },
        "48faa8b6ebaf4a38beee79d8f8561297": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "506b8bd646eb4a86b45785fca7854980": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50d082a0c56a434da92d46b06faf4183": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "556e9f565c904d27a1be0d97d0f7815e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5947c0fa58d44381a92be91b117dae5f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ea4b2d359394760b53ebbe3fc284ef2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5f718d11d839434a88bf8ff10f6dba8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "620a6da08dbc4f0d9237737675b16b07": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64251bda5720477b92415862eb0be923": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dc6aabeeabbf4b63b1a038c98ee298ff",
            "max": 425,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5f718d11d839434a88bf8ff10f6dba8d",
            "value": 425
          }
        },
        "67a8c60e2723447e8b07b754d3d8ae56": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9dfbb1dbddd44968c939048dde40b45",
            "placeholder": "​",
            "style": "IPY_MODEL_1bdccc0213cc453bb20e5e4abfd32e35",
            "value": " 248k/248k [00:00&lt;00:00, 10.7MB/s]"
          }
        },
        "7426e533f0fa47e1b984b64579567eb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_37308a8e76214ccc9f2cbef04caa38eb",
              "IPY_MODEL_a2edded1f3fb4a7fa5e56f061d2a66a0",
              "IPY_MODEL_011071963b2d4673bcc64a1bb1563946"
            ],
            "layout": "IPY_MODEL_adec0539c4b7498eb6ff0a13c4d088ef"
          }
        },
        "74c273e7e7b64242adc6004def38b843": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_abd98c227656448ca9a2c231f7d857d6",
            "placeholder": "​",
            "style": "IPY_MODEL_315211d6895c4d198b23f8ccf873b808",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "7975735bb20a4eff83e80edccaec76cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7abb2f60581b4a83bd86183d635cf70f",
              "IPY_MODEL_64251bda5720477b92415862eb0be923",
              "IPY_MODEL_0fca6111c86c439d9d1956c66c9d8bb4"
            ],
            "layout": "IPY_MODEL_f044aa8dcc7d447eb3decafa17ee8048"
          }
        },
        "7abb2f60581b4a83bd86183d635cf70f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9346162673844c39bf3f17f12a36323c",
            "placeholder": "​",
            "style": "IPY_MODEL_f2e976a2d7c249d5b4c76486ff3f66fc",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "7dbf3262548c4087801c93f8a7df5e09": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85557921902146b783ddf1eb4f81cf1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8b1fe2103510434cbd8476c41648a043": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9346162673844c39bf3f17f12a36323c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93a50faab46a40839612d729abea8fae": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea75731db26f40aeb96f3810f4d1b1c1",
            "placeholder": "​",
            "style": "IPY_MODEL_f4c43edd31a748ad998e3d80d0ce04d9",
            "value": " 289/289 [00:00&lt;00:00, 21.1kB/s]"
          }
        },
        "9614bd4a9a54499b95adaa3086ac19bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ea4b2d359394760b53ebbe3fc284ef2",
            "placeholder": "​",
            "style": "IPY_MODEL_219670845a3a42149f8d534a9d91a4ea",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "a20fdcd07a5d45b184f420650064a2e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5947c0fa58d44381a92be91b117dae5f",
            "max": 248477,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_85557921902146b783ddf1eb4f81cf1f",
            "value": 248477
          }
        },
        "a2edded1f3fb4a7fa5e56f061d2a66a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_506b8bd646eb4a86b45785fca7854980",
            "max": 494860,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_556e9f565c904d27a1be0d97d0f7815e",
            "value": 494860
          }
        },
        "abd1080af38a4e2894cf2a5cfa06fae4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_74c273e7e7b64242adc6004def38b843",
              "IPY_MODEL_edf3c9b38ba74ed1b2f93255c70882ba",
              "IPY_MODEL_93a50faab46a40839612d729abea8fae"
            ],
            "layout": "IPY_MODEL_310adfbc64b644c8b454a5d10cd30c39"
          }
        },
        "abd98c227656448ca9a2c231f7d857d6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "adec0539c4b7498eb6ff0a13c4d088ef": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b34136ec3fba4f539478a9bddb035bde": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bc1467abf51245b7a8b9f1745365893d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c89a9979474047838265db9e5b174bbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9614bd4a9a54499b95adaa3086ac19bb",
              "IPY_MODEL_a20fdcd07a5d45b184f420650064a2e7",
              "IPY_MODEL_67a8c60e2723447e8b07b754d3d8ae56"
            ],
            "layout": "IPY_MODEL_d5fcd237b1ab4e56b0ba26c66cea29ea"
          }
        },
        "d5fcd237b1ab4e56b0ba26c66cea29ea": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc6aabeeabbf4b63b1a038c98ee298ff": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e47c0c2f4d1e4605907c5828e0744c5e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7219bb4c46a4cb4b70ea67a4c6e3107": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9dfbb1dbddd44968c939048dde40b45": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea75731db26f40aeb96f3810f4d1b1c1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb27f6e1353c48318d075be6812799f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "edf3c9b38ba74ed1b2f93255c70882ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_620a6da08dbc4f0d9237737675b16b07",
            "max": 289,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8b1fe2103510434cbd8476c41648a043",
            "value": 289
          }
        },
        "f044aa8dcc7d447eb3decafa17ee8048": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2e571828f4945bf9285bf4de06cb886": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f2e976a2d7c249d5b4c76486ff3f66fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f4c43edd31a748ad998e3d80d0ce04d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9f64e83e6024bf38ee31dfaf0aa8f64": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}