code/NER4ID.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NER4ID at SemEval-2022 Task 2: Named Entity Recognition for Idiomaticity Detection\n",
    "### Tedeschi and Navigli (2022)\n",
    "----------------------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "xzZNVo0Bjn0t",
    "outputId": "2aa89a73-1185-44c2-a4c2-6496863d75bd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /root/miniconda3/envs/torch/lib/python3.8/site-packages (4.6.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (2022.3.15)\n",
      "Requirement already satisfied: numpy>=1.17 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (1.22.3)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (0.10.3)\n",
      "Requirement already satisfied: packaging in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: sacremoses in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (0.0.49)\n",
      "Requirement already satisfied: huggingface-hub==0.0.8 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (0.0.8)\n",
      "Requirement already satisfied: filelock in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: requests in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (2.27.1)\n",
      "Requirement already satisfied: tqdm>=4.27 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from transformers) (4.63.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from packaging->transformers) (3.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests->transformers) (2021.10.8)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests->transformers) (1.26.9)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests->transformers) (2.0.12)\n",
      "Requirement already satisfied: click in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from sacremoses->transformers) (8.0.4)\n",
      "Requirement already satisfied: six in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from sacremoses->transformers) (1.16.0)\n",
      "Requirement already satisfied: joblib in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from sacremoses->transformers) (1.1.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Requirement already satisfied: scikit-learn in /root/miniconda3/envs/torch/lib/python3.8/site-packages (1.0.2)\n",
      "Requirement already satisfied: numpy>=1.14.6 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from scikit-learn) (1.22.3)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from scikit-learn) (3.1.0)\n",
      "Requirement already satisfied: scipy>=1.1.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from scikit-learn) (1.8.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from scikit-learn) (1.1.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip install transformers\n",
    "! pip install scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "0q_SRDCMmrNk"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import Dataset\n",
    "from torch.utils.data.dataloader import DataLoader\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import torch.optim as optim\n",
    " \n",
    "from pprint import pprint\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from transformers import BertTokenizer, BertModel, BertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"../\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Q5rzN_LXmy3B",
    "outputId": "c790ac06-1a29-4d32-89f9-b59685283d52"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8.1+cu102\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/cuda/__init__.py:104: UserWarning: \n",
      "GeForce RTX 3090 with CUDA capability sm_86 is not compatible with the current PyTorch installation.\n",
      "The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_70.\n",
      "If you want to use the GeForce RTX 3090 GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/\n",
      "\n",
      "  warnings.warn(incompatible_device_warn.format(device_name, capability, \" \".join(arch_list), device_name))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'GeForce RTX 3090'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(torch.__version__)\n",
    "torch.cuda.current_device()\n",
    "torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "mode = input(\"Do you want to train a model from scratch or do you want to evaluate a pretrained model? Possible answers: ['train', 'eval']\\n> \")\n",
    "assert mode in ['train', 'eval']\n",
    "\n",
    "setting = input(\"\\nIn which setting do you want to train/evaluate your model? Possible answers: ['zero-shot', 'one-shot']\\n>\")\n",
    "assert setting in ['zero-shot', 'one-shot']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "1CBHdcuhnXBn"
   },
   "outputs": [],
   "source": [
    "train_file_zero_shot = \"data/train_zero_shot.csv\" \n",
    "train_file_one_shot = \"data/train_one_shot.csv\"\n",
    "\n",
    "dev_file = \"data/dev.csv\"\n",
    "dev_file_gold = \"data/dev_gold.csv\"\n",
    "test_file = \"data/test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DhxWcT1gGnuB",
    "outputId": "a8b24b7d-53ea-430d-fb2f-d37891fe5a53"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting en-core-web-sm==3.2.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl (13.9 MB)\n",
      "Requirement already satisfied: spacy<3.3.0,>=3.2.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from en-core-web-sm==3.2.0) (3.2.3)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (4.63.1)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (21.3)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.6.1)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.27.1)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: jinja2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.1.1)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.4.2)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.7)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.22.3)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.4.0)\n",
      "Requirement already satisfied: setuptools in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (58.0.4)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (8.0.15)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.8.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.0.6)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.3.0)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.7.7)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.9)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.6)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (0.9.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.0.7)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (5.2.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (4.1.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (1.26.9)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2021.10.8)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (8.0.4)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from jinja2->spacy<3.3.0,>=3.2.0->en-core-web-sm==3.2.0) (2.1.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n",
      "Collecting pt-core-news-sm==3.2.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/pt_core_news_sm-3.2.0/pt_core_news_sm-3.2.0-py3-none-any.whl (22.2 MB)\n",
      "Requirement already satisfied: spacy<3.3.0,>=3.2.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from pt-core-news-sm==3.2.0) (3.2.3)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2.27.1)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (1.22.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (21.3)\n",
      "Requirement already satisfied: setuptools in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (58.0.4)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (1.8.2)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (0.7.7)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2.4.2)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (3.3.0)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (1.0.6)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2.0.7)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.12 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (8.0.15)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (3.0.6)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (1.0.1)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (4.63.1)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (0.4.0)\n",
      "Requirement already satisfied: jinja2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (3.1.1)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (0.6.1)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.8.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (0.9.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2.0.6)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.8 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (3.0.9)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from packaging>=20.0->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (3.0.7)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from pathy>=0.3.5->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (5.2.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (4.1.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (1.26.9)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (3.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2021.10.8)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (8.0.4)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /root/miniconda3/envs/torch/lib/python3.8/site-packages (from jinja2->spacy<3.3.0,>=3.2.0->pt-core-news-sm==3.2.0) (2.1.1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('pt_core_news_sm')\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import en_core_web_sm\n",
    "import pt_core_news_sm\n",
    "from spacy.cli.download import download as spacy_download\n",
    "\n",
    "spacy_download(\"en_core_web_sm\")\n",
    "spacy_tagger_en = spacy.load(\"en_core_web_sm\", exclude=[\"ner\", \"parser\"])\n",
    "\n",
    "spacy_download(\"pt_core_news_sm\")\n",
    "spacy_tagger_pt = spacy.load(\"pt_core_news_sm\", exclude=[\"ner\", \"parser\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OJ4tYxBaoUhl",
    "outputId": "8af3b02d-4d28-43b5-fffc-d9c2cd9d1775"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4492it [00:00, 120087.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "train_data = {}\n",
    "idx = 0\n",
    "\n",
    "predictions = []\n",
    "labels = []\n",
    "\n",
    "if setting==\"zero-shot\" or setting==\"one-shot\":\n",
    "    with open(train_file_zero_shot, newline='') as csvfile:\n",
    "        reader = csv.reader(csvfile, delimiter=',')\n",
    "        for row in tqdm(reader):\n",
    "            \n",
    "            if len((row[4].strip() + row[5].strip() + row[6].strip()).split(\" \")) < 300:\n",
    "                text = row[4].strip()+ \" \" + row[5].strip()+ \" \" + row[6].strip()\n",
    "            elif len((row[4].strip() + row[5].strip()).split(\" \")) < 300:\n",
    "                text = row[4].strip()+ \" \" + row[5].strip()\n",
    "            else:\n",
    "                text = row[5].strip()\n",
    "                \n",
    "            e = row[2].strip()\n",
    "\n",
    "            train_data[idx] = {\n",
    "                                \"id\": row[0],\n",
    "                                \"lang\": row[1],\n",
    "                                \"expression\": e,\n",
    "                                \"text\": row[5].strip(),\n",
    "                                \"idiomatic\": True if row[7] == \"0\" else False\n",
    "                            }    \n",
    "            idx += 1\n",
    "\n",
    "\n",
    "if setting==\"one-shot\":\n",
    "    with open(train_file_one_shot, newline='') as csvfile:\n",
    "        reader = csv.reader(csvfile, delimiter=',')\n",
    "        for row in tqdm(reader):\n",
    "            \n",
    "            if len((row[4].strip() + row[5].strip() + row[6].strip()).split(\" \")) < 300:\n",
    "                text = row[4].strip()+ \" \" + row[5].strip()+ \" \" + row[6].strip()\n",
    "            elif len((row[4].strip() + row[5].strip()).split(\" \")) < 300:\n",
    "                text = row[4].strip()+ \" \" + row[5].strip()\n",
    "            else:\n",
    "                text = row[5].strip()\n",
    "                \n",
    "            e = row[2].strip()\n",
    "                        \n",
    "            train_data[idx] = {\n",
    "                                \"id\": row[0],\n",
    "                                \"lang\": row[1],\n",
    "                                \"expression\": e,\n",
    "                                \"text\": row[5].strip(),\n",
    "                                \"idiomatic\": True if row[7] == \"0\" else False\n",
    "                            }    \n",
    "            idx += 1\n",
    "\n",
    "print(len(train_data)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Q2kWaYAsyaS9",
    "outputId": "767ec0e8-57fe-4d69-dc00-5026131558f1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "740\n"
     ]
    }
   ],
   "source": [
    "labels_dev = {}\n",
    "\n",
    "with open(dev_file_gold, newline='') as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter=',')\n",
    "    for row in reader:\n",
    "        labels_dev[row[0]] = True if row[3] == \"0\" else False\n",
    "\n",
    "dev_data = {}\n",
    "idx = 0\n",
    "\n",
    "with open(dev_file, newline='') as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter=',')\n",
    "    for row in reader:\n",
    "\n",
    "        if len((row[3].strip() + row[4].strip() + row[5].strip()).split(\" \")) < 300:\n",
    "            text = row[3].strip() + \" \" + row[4].strip()+ \" \" + row[5].strip()\n",
    "        elif len((row[3].strip() + row[4].strip()).split(\" \")) < 300:\n",
    "            text = row[3].strip()+ \" \" + row[4].strip()\n",
    "        else:\n",
    "            text = row[4].strip()\n",
    "        \n",
    "        e = row[2].strip()\n",
    "                \n",
    "        \n",
    "        dev_data[idx] = {\n",
    "                        \"id\": row[0],\n",
    "                        \"lang\": row[1],\n",
    "                        \"expression\": e,\n",
    "                        \"text\": row[4].strip(),\n",
    "                        \"idiomatic\": labels_dev[row[0]]\n",
    "                        }\n",
    "                \n",
    "        idx += 1\n",
    "        \n",
    "    \n",
    "print(len(dev_data)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2343\n"
     ]
    }
   ],
   "source": [
    "test_data = {}\n",
    "idx = 0\n",
    "\n",
    "with open(test_file, newline='') as csvfile:\n",
    "    reader = csv.reader(csvfile, delimiter=',')\n",
    "    for row in reader:\n",
    "\n",
    "        if len((row[3].strip() + row[4].strip() + row[5].strip()).split(\" \")) < 300:\n",
    "            text = row[3].strip()+ \" \" + row[4].strip()+ \" \" + row[5].strip()\n",
    "        elif len((row[3].strip() + row[4].strip()).split(\" \")) < 300:\n",
    "            text = row[3].strip()+ \" \" + row[4].strip()\n",
    "        else:\n",
    "            text = row[4].strip()\n",
    "        \n",
    "        e = row[2].strip()\n",
    "\n",
    "        test_data[idx] = {\n",
    "                            \"id\": row[0],\n",
    "                            \"lang\": row[1],\n",
    "                            \"expression\": e,\n",
    "                            \"text\": row[4].strip(),\n",
    "                            \"idiomatic\": True #it's just a FAKE label used to maintain the same structure of the dataset entries, \n",
    "                                              #it could be also False\n",
    "                        }\n",
    "                \n",
    "        idx += 1\n",
    "        \n",
    "    \n",
    "print(len(test_data)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fix Random Seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "8v_-Re97pfxI"
   },
   "outputs": [],
   "source": [
    "SEED = 2 #we set a seed for having replicability of results\n",
    " \n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load BERT Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 284,
     "referenced_widgets": [
      "8ed578f71a004777904a58e8041ee9ce",
      "e3ad20224c694f51b458bab8abdac7cc",
      "ce55f77d26a44d15abe44cc7c8a860e1",
      "e2e7cd0a4f314f15ad919a2c3038e48a",
      "8f2d38ad495f4db1a2897dccde001644",
      "c0ed9bff08d147e49288b30b30277122",
      "76124438b06a43aba9df4a6f63655f74",
      "b372b8b46db44334a703b3e351444359",
      "1d33288b556847289d1066a65ae481bd",
      "75c95ba747d14199852c07180cd01daa",
      "16ad85d8b416474a81ebfca10118d768",
      "b64eed73d92c4bda8d37005b05036993",
      "d4e22ed7b3a44f7a9ea35e37c46241f5",
      "b15d395dc9cb41789a92de173280a45d",
      "8e925be421914723aef9b1285d5cd649",
      "d0cd29eb52354cc28a387d08eddcb95f",
      "f65ff8346d5f464a8de8af885e9fd5af",
      "920b320b004b4af0815cfc330fc3b10a",
      "a2135372eda5476eb64abdfedc646d71",
      "ed40f66ec75a4e68bb59aa1ccbdf31ad",
      "a31113a38b474be4bf7ea7c8d9c93ec7",
      "3dbd77f921ac4d579b8371d9aa7971dc",
      "b60c9e8bc6954b8eb3eaf07d362ef40b",
      "ff76d14aae6f446cbd1f0db79892acac",
      "876192ed31f94e74896d68f338313366",
      "1f8a3f2903e94de4ba00b3e9439eb10c",
      "cd2ce40542d746ec9184a34ef52351e7",
      "c12767e471194b2c809041f2a404f897",
      "f138c3c851c945cd94d92bdc008d58a6",
      "5bf5093e69e948c7af694b9a4e1b56da",
      "5361eb223338456dbb9c6bfba045167b",
      "734340c7f9aa434d84b63bd497ffa840",
      "8da747c2d3e84520b6d571fc8cc7e038",
      "444587ade2434752a004023f4b411394",
      "7d8fb2ecb5bf4fb49260b989d8cf04fc",
      "eaf72a9c6b2949d89098bdc33f30bd8f",
      "f0db6ccd5580479f87e6317c0b09f809",
      "c6b8e66a586f42318399c971f8df43f3",
      "fc82db4cec0c4b57a78fb4f338dbb718",
      "91fe74d5cf204f4884ce6429a3797c9b",
      "8ab8a9541b78463b860186ad0eaf7a82",
      "db8cb3de4cbc42fe93509a2ab0e51546",
      "fb6d4205bd9f42b1b2ea67ec84168eb4",
      "55576b6ba2da4258834f518edb0d708b",
      "bc204f9df229436ea1de9621db7e23ac",
      "772e9a8ce1d241069b11be3c33f90960",
      "ea6f8d1e7dd14ebb91f55252b9e58697",
      "2d0c44f052cc45308a380f4b8b1827cd",
      "8be0095322fd4962b6b82ea5b1186cf5",
      "5c76a256b616438eaadf8952d8135e3a",
      "e089514ffd9e48f2ae731a8bfe09124d",
      "354ed270170040f393f4df8fbac1a3d5",
      "9c4f669d083644939392617b09d5a585",
      "79960eb3196146399f88383f39ccc933",
      "cf63a36ae06f43628ebbaa2333d0e6e0"
     ]
    },
    "id": "fkTPuDGIpi30",
    "outputId": "1e4a7ea6-d79f-45cf-d9cf-56c09d402b76"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "model_name = 'bert-base-multilingual-cased'\n",
    " \n",
    "bert_config = BertConfig.from_pretrained(model_name, output_hidden_states=True)\n",
    "bert_tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "bert_model = BertModel.from_pretrained(model_name, config=bert_config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "bpd4vDOjprKv"
   },
   "outputs": [],
   "source": [
    "nlp_en = en_core_web_sm.load()\n",
    "nlp_pt = pt_core_news_sm.load()\n",
    "\n",
    "class IdiomDataset(Dataset):\n",
    "    def __init__(self, \n",
    "                 dataset, \n",
    "                 tokenizer,\n",
    "                 languages,\n",
    "                 device=\"cuda\",\n",
    "                ) -> None:\n",
    "        \n",
    "        self.encoded_data = []\n",
    "    \n",
    "        self.dataset = dataset\n",
    "        self.tokenizer = tokenizer\n",
    "        self.languages = languages\n",
    "        self.device = device\n",
    "        self.__init_encoded_data()\n",
    "\n",
    " \n",
    "    def __init_encoded_data(self):\n",
    "                \n",
    "        for idx in tqdm(self.dataset):\n",
    "            id = self.dataset[idx][\"id\"]\n",
    "            lang = self.dataset[idx][\"lang\"]\n",
    "            e = self.dataset[idx][\"expression\"]\n",
    "            label = self.dataset[idx][\"idiomatic\"]\n",
    "            context_tmp = self.dataset[idx][\"text\"]\n",
    "\n",
    "            if lang == \"EN\":\n",
    "                doc = nlp_en(context_tmp)\n",
    "                ents = [X.text.lower() for X in doc.ents]\n",
    "            else:\n",
    "                doc = nlp_pt(context_tmp)\n",
    "                ents = [X.text.lower() for X in doc.ents]\n",
    "            \n",
    "    \n",
    "            if e in ents:\n",
    "                e = context_tmp\n",
    "                context = context_tmp\n",
    "            else:\n",
    "                if context_tmp.find(e) != -1:\n",
    "                    context = context_tmp[:context_tmp.lower().find(e)] + context_tmp[context_tmp.lower().find(e)+len(e):]\n",
    "\n",
    "                else:    \n",
    "                    e = context_tmp\n",
    "                    context = context_tmp                     \n",
    "                \n",
    "        \n",
    "            tokenized_e = torch.tensor(self.tokenize_mention(e, self.tokenizer, True))\n",
    "            tokenized_context = torch.tensor(self.tokenize_mention(context, self.tokenizer, True))\n",
    "\n",
    "            if e!=\"MWE\" and lang in self.languages:\n",
    "                self.encoded_data.append((idx,\n",
    "                                          e,\n",
    "                                          context,\n",
    "                                          tokenized_e,\n",
    "                                          tokenized_context,\n",
    "                                          torch.tensor([-1.0]) if label == True else torch.tensor([1.0]),\n",
    "                                          id, \n",
    "                                          lang))\n",
    "\n",
    "     \n",
    "    def tokenize_mention(self, sent, tokenizer, special_tokens):\n",
    "        encoded_sentence = tokenizer.encode(sent, add_special_tokens = special_tokens)\n",
    "        return encoded_sentence[:500]\n",
    "    \n",
    "    def tokenize_description(self, sent, tokenizer, window):\n",
    "        encoded_sentence = tokenizer.encode(sent, add_special_tokens = True)\n",
    "        return encoded_sentence\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.encoded_data)\n",
    " \n",
    "    def __getitem__(self, idx: int):\n",
    "        return self.encoded_data[idx]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "aa72jvxrp8Na",
    "outputId": "d678f545-fff0-4583-a546-69bd20622cc3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4492/4492 [00:23<00:00, 187.69it/s]\n",
      "100%|██████████| 740/740 [00:03<00:00, 198.66it/s]\n",
      "100%|██████████| 2343/2343 [00:11<00:00, 201.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4491\n",
      "739\n",
      "2342\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_dataset = IdiomDataset(train_data, bert_tokenizer, languages = [\"EN\", \"PT\", \"GL\"])\n",
    "dev_dataset = IdiomDataset(dev_data, bert_tokenizer, languages =[\"EN\", \"PT\", \"GL\"])\n",
    "test_dataset = IdiomDataset(test_data, bert_tokenizer, languages = [\"EN\",\"PT\", \"GL\"])\n",
    "\n",
    "print(len(train_dataset))\n",
    "print(len(dev_dataset))\n",
    "print(len(test_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the dataloader:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bfL32r72qBWh",
    "outputId": "b492727f-af78-4a5c-e3a4-2d0161d71c94"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "562\n",
      "93\n",
      "293\n"
     ]
    }
   ],
   "source": [
    "def collate(elems: tuple) -> tuple:\n",
    "    ids, e, texts, expressions, contexts, labels, ids, langs = list(zip(*elems))\n",
    "    \n",
    "    pad_expressions = pad_sequence(expressions, batch_first=True, padding_value=0)\n",
    "    pad_contexts = pad_sequence(contexts, batch_first=True, padding_value=0)\n",
    "    pad_labels = pad_sequence(labels, batch_first=True, padding_value=0)\n",
    " \n",
    "    return ids, e, texts, pad_expressions, pad_contexts, pad_labels.cuda(), ids, langs\n",
    "\n",
    "\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate)\n",
    "dev_dataloader = DataLoader(dev_dataset, batch_size=8, shuffle=False, collate_fn=collate)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=8, shuffle=False, collate_fn=collate)\n",
    "\n",
    "print(len(train_dataloader))\n",
    "print(len(dev_dataloader))\n",
    "print(len(test_dataloader))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dual-Encoder Architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "cfUxreHMpx4E"
   },
   "outputs": [],
   "source": [
    "class DualEncoder(nn.Module):\n",
    "    def __init__(self, hparams):\n",
    "        super(DualEncoder, self).__init__()\n",
    "        pprint(params)\n",
    " \n",
    "        self.hparams = hparams\n",
    " \n",
    "        self.expression_encoder = BertModel.from_pretrained(model_name, config=bert_config)\n",
    "        self.context_encoder = BertModel.from_pretrained(model_name, config=bert_config)\n",
    "\n",
    "        self.cosine_similarity = nn.CosineSimilarity(dim=-1, eps=1e-6)\n",
    "        \n",
    "        self.dropout = nn.Dropout(hparams.dropout)\n",
    "\n",
    "        self.tanh = nn.Tanh()\n",
    "            \n",
    "        for param in self.context_encoder.parameters():\n",
    "            param.requires_grad = False\n",
    "  \n",
    "\n",
    "    def forward(self, expression, context, mask1, mask2):\n",
    "        \n",
    "        embedding_context = self.context_encoder.forward(context.cuda(), mask2.cuda())[0]\n",
    "        embedding_context = embedding_context[:,0,:].squeeze(1)\n",
    "\n",
    "        embedding_expression = self.expression_encoder.forward(expression.cuda(), mask1.cuda())[0]\n",
    "        embedding_expression = torch.sum(embedding_expression, 1)\n",
    "            \n",
    "        similarities = self.cosine_similarity(embedding_expression, embedding_context) \n",
    "                        \n",
    "        return similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HParams():\n",
    "    dropout = 0.25\n",
    "    \n",
    "params = HParams()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiate the model: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<__main__.HParams object at 0x7f5cbda7b0a0>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DualEncoder(\n",
       "  (expression_encoder): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (context_encoder): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (cosine_similarity): CosineSimilarity()\n",
       "  (dropout): Dropout(p=0.25, inplace=False)\n",
       "  (tanh): Tanh()\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_model = DualEncoder(params).cuda()\n",
    "my_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "P0JfvAWUp6gc"
   },
   "outputs": [],
   "source": [
    "class Trainer():\n",
    "    def __init__(self,\n",
    "                model:nn.Module, \n",
    "                loss_function,\n",
    "                optimizer,\n",
    "                gradient_accumulation_steps):\n",
    "        \n",
    "        self.model = model\n",
    "        self.loss_function = loss_function\n",
    "        self.optimizer = optimizer\n",
    "        self.gradient_accumulation_steps = gradient_accumulation_steps\n",
    " \n",
    "    def padding_mask(self, batch):\n",
    "        padding = torch.ones_like(batch)\n",
    "        padding[batch == 0] = 0\n",
    "        padding = padding.type(torch.uint8)\n",
    "        return padding\n",
    " \n",
    "    def train(self,\n",
    "            train_dataset:Dataset, \n",
    "            valid_dataset:Dataset,\n",
    "            epochs:int=1,\n",
    "            patience:int=10,\n",
    "            modelname=setting):\n",
    "        \n",
    "        print(\"\\nTraining...\")\n",
    " \n",
    "        record_dev = 0.0\n",
    "        full_patience = patience\n",
    "        modelname = modelname\n",
    " \n",
    "        for epoch in range(epochs):\n",
    "             if patience>0:\n",
    "                print(\" Epoch {:03d}\".format(epoch + 1))\n",
    "\n",
    "                epoch_loss = 0.0\n",
    "                self.model.train()\n",
    "                \n",
    "                count_batches = 0\n",
    "                self.optimizer.zero_grad()\n",
    "                \n",
    "                for ids, e, text, expressions, contexts, labels, id, lang in tqdm(train_dataset):\n",
    "                    batch_loss = 0.0\n",
    "\n",
    "                    mask1 = self.padding_mask(expressions)\n",
    "                    mask2 = self.padding_mask(contexts)\n",
    "                    \n",
    "                    similarities = self.model(expressions, contexts, mask1, mask2)\n",
    "\n",
    "                    labels = labels.view(-1)\n",
    "                    batch_loss = self.loss_function(similarities, labels)\n",
    "                    epoch_loss += batch_loss\n",
    "\n",
    "                    batch_loss.backward()\n",
    "\n",
    "                    if count_batches % self.gradient_accumulation_steps == 0:\n",
    "                        self.optimizer.step()\n",
    "                        self.optimizer.zero_grad()\n",
    "\n",
    "\n",
    "                avg_epoch_loss = epoch_loss / len(train_dataset)\n",
    "                print('[E: {:2d}] train loss = {:0.4f}'.format(epoch+1, avg_epoch_loss))\n",
    "\n",
    "                valid_loss, f1 = self.evaluate(valid_dataset, epoch)\n",
    "\n",
    "                if f1>record_dev:\n",
    "                    record_dev = f1\n",
    "                    torch.save(self.model.state_dict(), \"./checkpoints/\"+modelname+\".pt\")\n",
    "                    patience = full_patience\n",
    "                else:\n",
    "                    patience -= 1\n",
    "                   \n",
    "                print('\\t[E: {:2d}] valid loss = {:0.4f}, f1-score = {:0.4f}, patience: {:2d}'.format(epoch+1, valid_loss, f1, patience))\n",
    "\n",
    "\n",
    "        print(\"...Done!\")\n",
    "\n",
    "        return avg_epoch_loss\n",
    "\n",
    "    def evaluate(self, valid_dataset, epoch):\n",
    "\n",
    "        valid_loss = 0.0\n",
    "        all_predictions = list()\n",
    "        all_labels = list()\n",
    "        \n",
    "        predictions = {}\n",
    "         \n",
    "        self.model.eval()\n",
    "                    \n",
    "        for ids, e, text, expressions, contexts, labels, id, lang in tqdm(valid_dataset):\n",
    "            mask1 = self.padding_mask(expressions)\n",
    "            mask2 = self.padding_mask(contexts)\n",
    "            \n",
    "            with torch.no_grad():\n",
    "                similarities = self.model(expressions, contexts, mask1, mask2)\n",
    "\n",
    "            labels = labels.view(-1)\n",
    "            loss = self.loss_function(similarities, labels)\n",
    " \n",
    "            for i in range(len(similarities)):\n",
    "                if similarities[i]>0:\n",
    "                    all_predictions.append(1)\n",
    "                else:\n",
    "                    all_predictions.append(0)\n",
    "\n",
    "                all_labels.append(1 if labels[i].item()==1 else 0)\n",
    "            \n",
    "            valid_loss += loss\n",
    "            \n",
    "            for i in range(len(similarities)):\n",
    "                if similarities[i]>0:\n",
    "                    predictions[ids[i]] = 1\n",
    "\n",
    "                else:\n",
    "                    predictions[ids[i]] = 0\n",
    "\n",
    "        f1 = f1_score(all_labels, all_predictions, average= 'macro')\n",
    "        print(classification_report(all_labels, all_predictions, digits=4))\n",
    "        \n",
    "        return valid_loss / len(valid_dataset), f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiate the trainer:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "AJOGLgoZqR9w"
   },
   "outputs": [],
   "source": [
    "trainer = Trainer(model = my_model,\n",
    "                    loss_function = nn.MSELoss(),\n",
    "                    optimizer = optim.Adam(my_model.parameters(), lr=0.00001),\n",
    "                    gradient_accumulation_steps=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train the system:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "m0XLLlyoqU5_",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "if mode == \"train\":\n",
    "    trainer.train(train_dataloader, dev_dataloader, epochs = 1000, patience = 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict the test set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the best model checkpoint:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'checkpoints/zero-shot.pt'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/mnt/data/ner4id/code/NER4ID.ipynb Cell 37'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2B192.168.1.168/mnt/data/ner4id/code/NER4ID.ipynb#ch0000036vscode-remote?line=0'>1</a>\u001b[0m my_model\u001b[39m.\u001b[39mload_state_dict(torch\u001b[39m.\u001b[39;49mload(\u001b[39mf\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mcheckpoints/\u001b[39;49m\u001b[39m{\u001b[39;49;00msetting\u001b[39m}\u001b[39;49;00m\u001b[39m.pt\u001b[39;49m\u001b[39m\"\u001b[39;49m))\n",
      "File \u001b[0;32m~/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py:579\u001b[0m, in \u001b[0;36mload\u001b[0;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=575'>576</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m'\u001b[39m\u001b[39mencoding\u001b[39m\u001b[39m'\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m pickle_load_args\u001b[39m.\u001b[39mkeys():\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=576'>577</a>\u001b[0m     pickle_load_args[\u001b[39m'\u001b[39m\u001b[39mencoding\u001b[39m\u001b[39m'\u001b[39m] \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mutf-8\u001b[39m\u001b[39m'\u001b[39m\n\u001b[0;32m--> <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=578'>579</a>\u001b[0m \u001b[39mwith\u001b[39;00m _open_file_like(f, \u001b[39m'\u001b[39;49m\u001b[39mrb\u001b[39;49m\u001b[39m'\u001b[39;49m) \u001b[39mas\u001b[39;00m opened_file:\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=579'>580</a>\u001b[0m     \u001b[39mif\u001b[39;00m _is_zipfile(opened_file):\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=580'>581</a>\u001b[0m         \u001b[39m# The zipfile reader is going to advance the current file position.\u001b[39;00m\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=581'>582</a>\u001b[0m         \u001b[39m# If we want to actually tail call to torch.jit.load, we need to\u001b[39;00m\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=582'>583</a>\u001b[0m         \u001b[39m# reset back to the original position.\u001b[39;00m\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=583'>584</a>\u001b[0m         orig_position \u001b[39m=\u001b[39m opened_file\u001b[39m.\u001b[39mtell()\n",
      "File \u001b[0;32m~/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py:230\u001b[0m, in \u001b[0;36m_open_file_like\u001b[0;34m(name_or_buffer, mode)\u001b[0m\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=227'>228</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_open_file_like\u001b[39m(name_or_buffer, mode):\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=228'>229</a>\u001b[0m     \u001b[39mif\u001b[39;00m _is_path(name_or_buffer):\n\u001b[0;32m--> <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=229'>230</a>\u001b[0m         \u001b[39mreturn\u001b[39;00m _open_file(name_or_buffer, mode)\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=230'>231</a>\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=231'>232</a>\u001b[0m         \u001b[39mif\u001b[39;00m \u001b[39m'\u001b[39m\u001b[39mw\u001b[39m\u001b[39m'\u001b[39m \u001b[39min\u001b[39;00m mode:\n",
      "File \u001b[0;32m~/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py:211\u001b[0m, in \u001b[0;36m_open_file.__init__\u001b[0;34m(self, name, mode)\u001b[0m\n\u001b[1;32m    <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=209'>210</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__init__\u001b[39m(\u001b[39mself\u001b[39m, name, mode):\n\u001b[0;32m--> <a href='file:///root/miniconda3/envs/torch/lib/python3.8/site-packages/torch/serialization.py?line=210'>211</a>\u001b[0m     \u001b[39msuper\u001b[39m(_open_file, \u001b[39mself\u001b[39m)\u001b[39m.\u001b[39m\u001b[39m__init__\u001b[39m(\u001b[39mopen\u001b[39;49m(name, mode))\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'checkpoints/zero-shot.pt'"
     ]
    }
   ],
   "source": [
    "my_model.load_state_dict(torch.load(f\"checkpoints/{setting}.pt\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and then use the model to predict the test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "fjt7AH1FvXyo",
    "outputId": "c74ac723-3070-4ac4-9121-be8083debaa6"
   },
   "outputs": [],
   "source": [
    "def padding_mask(batch):\n",
    "        padding = torch.ones_like(batch)\n",
    "        padding[batch == 0] = 0\n",
    "        padding = padding.type(torch.uint8)\n",
    "        return padding\n",
    "\n",
    "def predict(model, test_dataset):\n",
    "\n",
    "    predictions = {}\n",
    "        \n",
    "    model.eval()\n",
    "                \n",
    "    for ids, e, text, expressions, contexts, labels, id, lang in tqdm(test_dataset):\n",
    "        mask1 = padding_mask(expressions)\n",
    "        mask2 = padding_mask(contexts)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            similarities = model(expressions, contexts, mask1, mask2)\n",
    "\n",
    "        labels = labels.view(-1)\n",
    "\n",
    "        for i in range(len(similarities)):\n",
    "            if similarities[i]>0:\n",
    "                predictions[ids[i]] = 1\n",
    "\n",
    "            else:\n",
    "                predictions[ids[i]] = 0\n",
    "\n",
    "\n",
    "    predictions[\"ID\"] = \"Label\"\n",
    "\n",
    "    if setting == \"zero-shot\":\n",
    "        with open(\"data/test_submission_format.csv\") as csvfile:\n",
    "            with open(\"predictions/task2_subtaska.csv\", \"w\") as out:\n",
    "                reader = csv.reader(csvfile, delimiter=',')\n",
    "                for row in reader:\n",
    "                    if row[2] == \"zero_shot\" or row[0] == \"ID\":\n",
    "                        out.write(row[0] + \",\" + row[1] + \",\" + row[2] + \",\" + str(predictions[row[0]]) + \"\\n\")\n",
    "                    else:\n",
    "                        out.write(row[0] + \",\" + row[1] + \",\" + row[2] + \",\" + \"\" + \"\\n\")\n",
    "    else:\n",
    "        with open(\"data/test_submission_format.csv\") as csvfile:\n",
    "            with open(\"predictions/task2_subtaska.csv\", \"w\") as out:\n",
    "                reader = csv.reader(csvfile, delimiter=',')\n",
    "                for row in reader:\n",
    "                    if row[2] == \"one_shot\" or row[0] == \"ID\":\n",
    "                        out.write(row[0] + \",\" + row[1] + \",\" + row[2] + \",\" + str(predictions[row[0]]) + \"\\n\")\n",
    "                    else:\n",
    "                        out.write(row[0] + \",\" + row[1] + \",\" + row[2] + \",\" + \"\" + \"\\n\")    \n",
    "\n",
    "\n",
    "predict(my_model, test_dataloader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The output file (saved in the /predictions folder) contains predictions in the standard format specified by the competition rules. You can now upload the output file on CodaLab and see the scores obtained by the system on the test set."
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "SemEval-2022.ipynb",
   "provenance": []
  },
  "interpreter": {
   "hash": "6f64be793538f7fe230f350828c9baf03d97c4df0981f52e8388f53f367f4a42"
  },
  "kernelspec": {
   "display_name": "Python 3.8.12 ('torch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "16ad85d8b416474a81ebfca10118d768": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1d33288b556847289d1066a65ae481bd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1f8a3f2903e94de4ba00b3e9439eb10c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5361eb223338456dbb9c6bfba045167b",
      "max": 29,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_5bf5093e69e948c7af694b9a4e1b56da",
      "value": 29
     }
    },
    "2d0c44f052cc45308a380f4b8b1827cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9c4f669d083644939392617b09d5a585",
      "max": 714314041,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_354ed270170040f393f4df8fbac1a3d5",
      "value": 714314041
     }
    },
    "354ed270170040f393f4df8fbac1a3d5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "3dbd77f921ac4d579b8371d9aa7971dc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "444587ade2434752a004023f4b411394": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_eaf72a9c6b2949d89098bdc33f30bd8f",
       "IPY_MODEL_f0db6ccd5580479f87e6317c0b09f809",
       "IPY_MODEL_c6b8e66a586f42318399c971f8df43f3"
      ],
      "layout": "IPY_MODEL_7d8fb2ecb5bf4fb49260b989d8cf04fc"
     }
    },
    "5361eb223338456dbb9c6bfba045167b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "55576b6ba2da4258834f518edb0d708b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5bf5093e69e948c7af694b9a4e1b56da": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "5c76a256b616438eaadf8952d8135e3a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "734340c7f9aa434d84b63bd497ffa840": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "75c95ba747d14199852c07180cd01daa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "76124438b06a43aba9df4a6f63655f74": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "772e9a8ce1d241069b11be3c33f90960": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "79960eb3196146399f88383f39ccc933": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "7d8fb2ecb5bf4fb49260b989d8cf04fc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "876192ed31f94e74896d68f338313366": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f138c3c851c945cd94d92bdc008d58a6",
      "placeholder": "​",
      "style": "IPY_MODEL_c12767e471194b2c809041f2a404f897",
      "value": "Downloading: 100%"
     }
    },
    "8ab8a9541b78463b860186ad0eaf7a82": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "8be0095322fd4962b6b82ea5b1186cf5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_cf63a36ae06f43628ebbaa2333d0e6e0",
      "placeholder": "​",
      "style": "IPY_MODEL_79960eb3196146399f88383f39ccc933",
      "value": " 681M/681M [00:27&lt;00:00, 29.3MB/s]"
     }
    },
    "8da747c2d3e84520b6d571fc8cc7e038": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8e925be421914723aef9b1285d5cd649": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ed40f66ec75a4e68bb59aa1ccbdf31ad",
      "max": 995526,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_a2135372eda5476eb64abdfedc646d71",
      "value": 995526
     }
    },
    "8ed578f71a004777904a58e8041ee9ce": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ce55f77d26a44d15abe44cc7c8a860e1",
       "IPY_MODEL_e2e7cd0a4f314f15ad919a2c3038e48a",
       "IPY_MODEL_8f2d38ad495f4db1a2897dccde001644"
      ],
      "layout": "IPY_MODEL_e3ad20224c694f51b458bab8abdac7cc"
     }
    },
    "8f2d38ad495f4db1a2897dccde001644": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_16ad85d8b416474a81ebfca10118d768",
      "placeholder": "​",
      "style": "IPY_MODEL_75c95ba747d14199852c07180cd01daa",
      "value": " 625/625 [00:00&lt;00:00, 7.04kB/s]"
     }
    },
    "91fe74d5cf204f4884ce6429a3797c9b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "920b320b004b4af0815cfc330fc3b10a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9c4f669d083644939392617b09d5a585": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a2135372eda5476eb64abdfedc646d71": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "a31113a38b474be4bf7ea7c8d9c93ec7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b15d395dc9cb41789a92de173280a45d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_920b320b004b4af0815cfc330fc3b10a",
      "placeholder": "​",
      "style": "IPY_MODEL_f65ff8346d5f464a8de8af885e9fd5af",
      "value": "Downloading: 100%"
     }
    },
    "b372b8b46db44334a703b3e351444359": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "b60c9e8bc6954b8eb3eaf07d362ef40b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_876192ed31f94e74896d68f338313366",
       "IPY_MODEL_1f8a3f2903e94de4ba00b3e9439eb10c",
       "IPY_MODEL_cd2ce40542d746ec9184a34ef52351e7"
      ],
      "layout": "IPY_MODEL_ff76d14aae6f446cbd1f0db79892acac"
     }
    },
    "b64eed73d92c4bda8d37005b05036993": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b15d395dc9cb41789a92de173280a45d",
       "IPY_MODEL_8e925be421914723aef9b1285d5cd649",
       "IPY_MODEL_d0cd29eb52354cc28a387d08eddcb95f"
      ],
      "layout": "IPY_MODEL_d4e22ed7b3a44f7a9ea35e37c46241f5"
     }
    },
    "bc204f9df229436ea1de9621db7e23ac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ea6f8d1e7dd14ebb91f55252b9e58697",
       "IPY_MODEL_2d0c44f052cc45308a380f4b8b1827cd",
       "IPY_MODEL_8be0095322fd4962b6b82ea5b1186cf5"
      ],
      "layout": "IPY_MODEL_772e9a8ce1d241069b11be3c33f90960"
     }
    },
    "c0ed9bff08d147e49288b30b30277122": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c12767e471194b2c809041f2a404f897": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c6b8e66a586f42318399c971f8df43f3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_55576b6ba2da4258834f518edb0d708b",
      "placeholder": "​",
      "style": "IPY_MODEL_fb6d4205bd9f42b1b2ea67ec84168eb4",
      "value": " 1.87M/1.87M [00:00&lt;00:00, 2.44MB/s]"
     }
    },
    "cd2ce40542d746ec9184a34ef52351e7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8da747c2d3e84520b6d571fc8cc7e038",
      "placeholder": "​",
      "style": "IPY_MODEL_734340c7f9aa434d84b63bd497ffa840",
      "value": " 29.0/29.0 [00:00&lt;00:00, 687B/s]"
     }
    },
    "ce55f77d26a44d15abe44cc7c8a860e1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_76124438b06a43aba9df4a6f63655f74",
      "placeholder": "​",
      "style": "IPY_MODEL_c0ed9bff08d147e49288b30b30277122",
      "value": "Downloading: 100%"
     }
    },
    "cf63a36ae06f43628ebbaa2333d0e6e0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d0cd29eb52354cc28a387d08eddcb95f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3dbd77f921ac4d579b8371d9aa7971dc",
      "placeholder": "​",
      "style": "IPY_MODEL_a31113a38b474be4bf7ea7c8d9c93ec7",
      "value": " 972k/972k [00:00&lt;00:00, 1.84MB/s]"
     }
    },
    "d4e22ed7b3a44f7a9ea35e37c46241f5": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "db8cb3de4cbc42fe93509a2ab0e51546": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e089514ffd9e48f2ae731a8bfe09124d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e2e7cd0a4f314f15ad919a2c3038e48a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1d33288b556847289d1066a65ae481bd",
      "max": 625,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b372b8b46db44334a703b3e351444359",
      "value": 625
     }
    },
    "e3ad20224c694f51b458bab8abdac7cc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ea6f8d1e7dd14ebb91f55252b9e58697": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e089514ffd9e48f2ae731a8bfe09124d",
      "placeholder": "​",
      "style": "IPY_MODEL_5c76a256b616438eaadf8952d8135e3a",
      "value": "Downloading: 100%"
     }
    },
    "eaf72a9c6b2949d89098bdc33f30bd8f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_91fe74d5cf204f4884ce6429a3797c9b",
      "placeholder": "​",
      "style": "IPY_MODEL_fc82db4cec0c4b57a78fb4f338dbb718",
      "value": "Downloading: 100%"
     }
    },
    "ed40f66ec75a4e68bb59aa1ccbdf31ad": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f0db6ccd5580479f87e6317c0b09f809": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_db8cb3de4cbc42fe93509a2ab0e51546",
      "max": 1961828,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8ab8a9541b78463b860186ad0eaf7a82",
      "value": 1961828
     }
    },
    "f138c3c851c945cd94d92bdc008d58a6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f65ff8346d5f464a8de8af885e9fd5af": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fb6d4205bd9f42b1b2ea67ec84168eb4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "fc82db4cec0c4b57a78fb4f338dbb718": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ff76d14aae6f446cbd1f0db79892acac": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}