Masked Language Model/Datasets for NLP - Reddit Comments/Datasets_for_NLP_MLM.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9-R0hkELQI8b"
      },
      "source": [
        "## Reddit Comments: Masked Language Modeling\n",
        "\n",
        "Dataset Source: https://www.kaggle.com/datasets/toygarr/datasets-for-natural-language-processing"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Install Missing Libraries"
      ],
      "metadata": {
        "id": "YxM4-xm6AVKR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install datasets transformers torch"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jLUVJKW5AVCu",
        "outputId": "bb1307e0-3e10-4c3d-8d41-78c15f6bb7f4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.10.1-py3-none-any.whl (469 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 KB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers\n",
            "  Downloading transformers-4.27.0-py3-none-any.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m36.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (1.13.1+cu116)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (4.65.0)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (2023.3.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets) (1.22.4)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (9.0.0)\n",
            "Collecting dill<0.3.7,>=0.3.0\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 KB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiohttp\n",
            "  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets) (23.0)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets) (2.25.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets) (1.4.4)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 KB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting huggingface-hub<1.0.0,>=0.2.0\n",
            "  Downloading huggingface_hub-0.13.2-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.2/199.2 KB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 KB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets) (6.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch) (4.5.0)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting charset-normalizer<4.0,>=2.0\n",
            "  Downloading charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.2/199.2 KB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: tokenizers, xxhash, multidict, frozenlist, dill, charset-normalizer, async-timeout, yarl, responses, multiprocess, huggingface-hub, aiosignal, transformers, aiohttp, datasets\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 charset-normalizer-3.1.0 datasets-2.10.1 dill-0.3.6 frozenlist-1.3.3 huggingface-hub-0.13.2 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 tokenizers-0.13.2 transformers-4.27.0 xxhash-3.2.0 yarl-1.8.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Import Necessary Libraries"
      ],
      "metadata": {
        "id": "03HKuEJtA6QP"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "a2LYD421QI8d",
        "outputId": "28e86ffe-e31e-43b7-f793-ec278b89a1f0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Error: Failed to call git rev-parse --git-dir: exit status 128 \n",
            "Git LFS initialized.\n"
          ]
        }
      ],
      "source": [
        "import os, sys, random, math\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import torch\n",
        "\n",
        "import datasets\n",
        "from datasets import ClassLabel, Dataset, DatasetDict\n",
        "\n",
        "import transformers\n",
        "from transformers import AutoTokenizer, AutoModelForMaskedLM, DataCollatorForLanguageModeling\n",
        "from transformers import TrainingArguments, Trainer, set_seed\n",
        "\n",
        "!git lfs install"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### HuggingFace Access"
      ],
      "metadata": {
        "id": "5kPdIQQEAm9f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli login"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HQ1XTiTYAmIZ",
        "outputId": "468f8be2-5750-4705-8f64-ae488e407865"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "    _|    _|  _|    _|    _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|_|_|_|    _|_|      _|_|_|  _|_|_|_|\n",
            "    _|    _|  _|    _|  _|        _|          _|    _|_|    _|  _|            _|        _|    _|  _|        _|\n",
            "    _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|\n",
            "    _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|\n",
            "    _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|\n",
            "    \n",
            "    To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n",
            "Token: \n",
            "Add token as git credential? (Y/n) Y\n",
            "Token is valid.\n",
            "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n",
            "You might have to re-authenticate when pushing to the Hugging Face Hub.\n",
            "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n",
            "\n",
            "git config --global credential.helper store\n",
            "\n",
            "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n",
            "Token has not been saved to git credential helper.\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Mount Google Drive"
      ],
      "metadata": {
        "id": "3Px7G56yAtRv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/drive\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "39uwpTHmAtJe",
        "outputId": "b45b484a-1f2f-4026-e496-aa943332de98"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gt36jW4YQI8e"
      },
      "source": [
        "#### Display Library Versions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "r_Md8BStQI8e",
        "outputId": "026f9454-cf1b-4bbb-f13e-3d44c4b3296b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "        Python: 3.9.16\n",
            "        Pandas: 1.4.4\n",
            "      Datasets: 2.10.1\n",
            "  Transformers: 4.27.0\n"
          ]
        }
      ],
      "source": [
        "print(\"Python:\".rjust(15), sys.version[0:6])\n",
        "print(\"Pandas:\".rjust(15), pd.__version__)\n",
        "print(\"Datasets:\".rjust(15), datasets.__version__)\n",
        "print(\"Transformers:\".rjust(15), transformers.__version__)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rDzZjeNPQI8e"
      },
      "source": [
        "#### Load Training Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "zEVUk6tUQI8e",
        "outputId": "38917142-9e38-4484-9bb2-e49e838407de",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                     text\n",
              "0       I don't know man, that Jeep looks really cool ...\n",
              "1       Bring back old wits end and lets see how long ...\n",
              "2       I don't think he's listening, he's more focuse...\n",
              "3       Either that or we are all being tricked by the...\n",
              "4                                          You forgot the\n",
              "...                                                   ...\n",
              "707536              If I will look like that, sign me up!\n",
              "707537                             *\"fighting terrorism\"*\n",
              "707538  Idk man, if I had a really cool sticker worth ...\n",
              "707539                     For starters I recommend arch.\n",
              "707540                              Someone make this IRL\n",
              "\n",
              "[676518 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-949918ec-2f37-4947-bef0-807eb071c032\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I don't know man, that Jeep looks really cool ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Bring back old wits end and lets see how long ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>I don't think he's listening, he's more focuse...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Either that or we are all being tricked by the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>You forgot the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>707536</th>\n",
              "      <td>If I will look like that, sign me up!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>707537</th>\n",
              "      <td>*\"fighting terrorism\"*</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>707538</th>\n",
              "      <td>Idk man, if I had a really cool sticker worth ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>707539</th>\n",
              "      <td>For starters I recommend arch.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>707540</th>\n",
              "      <td>Someone make this IRL</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>676518 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-949918ec-2f37-4947-bef0-807eb071c032')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-949918ec-2f37-4947-bef0-807eb071c032 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-949918ec-2f37-4947-bef0-807eb071c032');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "parent_dir = \"/content/drive/MyDrive\"\n",
        "\n",
        "train_data_file = os.path.join(parent_dir, \"train.csv\")\n",
        "\n",
        "train_data = pd.read_csv(train_data_file, usecols=[\"text\"])\n",
        "\n",
        "train_data = train_data.drop_duplicates()\n",
        "train_data = train_data.dropna()\n",
        "\n",
        "train_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7oGt6okRQI8f"
      },
      "source": [
        "#### Load Evaluation Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "PcjSNqCMQI8f",
        "outputId": "675ebc65-8281-4385-ea5b-3f3d41b5c0d0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                     text\n",
              "0              Huh, I didn't know ESO HAD male Spriggans.\n",
              "1                              Because cats have 9 lives.\n",
              "2                                      Not possible, duh.\n",
              "3       i5 3570k, 8GB Ram, GeForce 660TI, 1TB mechanic...\n",
              "4       I assume it's a factory 1000 bigger than a Meg...\n",
              "...                                                   ...\n",
              "303227  Respect boundaries, don't push them or try to ...\n",
              "303228                     Damn that makes a lot of sense\n",
              "303229     What's the mt Vernon Thai place you'd suggest?\n",
              "303230                               God damn it Connolly\n",
              "303231  File this under \"most cringe-worthy things sai...\n",
              "\n",
              "[292927 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b2b49538-d2f5-43d4-b245-ed7de6735863\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Huh, I didn't know ESO HAD male Spriggans.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Because cats have 9 lives.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Not possible, duh.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>i5 3570k, 8GB Ram, GeForce 660TI, 1TB mechanic...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I assume it's a factory 1000 bigger than a Meg...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>303227</th>\n",
              "      <td>Respect boundaries, don't push them or try to ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>303228</th>\n",
              "      <td>Damn that makes a lot of sense</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>303229</th>\n",
              "      <td>What's the mt Vernon Thai place you'd suggest?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>303230</th>\n",
              "      <td>God damn it Connolly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>303231</th>\n",
              "      <td>File this under \"most cringe-worthy things sai...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>292927 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b2b49538-d2f5-43d4-b245-ed7de6735863')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b2b49538-d2f5-43d4-b245-ed7de6735863 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b2b49538-d2f5-43d4-b245-ed7de6735863');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "eval_data_file = os.path.join(parent_dir, \"test.csv\")\n",
        "\n",
        "eval_data = pd.read_csv(eval_data_file, usecols=[\"text\"])\n",
        "\n",
        "eval_data = eval_data.drop_duplicates()\n",
        "eval_data = eval_data.dropna()\n",
        "\n",
        "eval_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1EHy6DzPQI8f"
      },
      "source": [
        "#### Visual: Histogram of Input Lengths"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "FSMHOniWQI8f",
        "outputId": "1a9a9393-07dc-4c2c-b856-e6cfd8a71d52",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The maximum length of this dataset is approximately 2222 words.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1296x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABCEAAAHwCAYAAACYKHXTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAp1ElEQVR4nO3dfdRmZ10f+u/PBBB5S4AxJ00CgxDFQEskMQRfKILNC9UmdiENWhNpDlEJHvTYVQfb01goHjhHRFIhFmUk8S2mtEgkwTiNWBUIZngxIQSaEYhJzBtMQkAUGvidP+5rDjfDMzPPJDPXMzP5fNa6173377r23tf9JHvdz/Oda+9d3R0AAACAve3r1noAAAAAwAODEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQALCGquq6qnr2Wo9jLVXVD1TVTVX1uar6trUez76iqrqqnrTW4wCAPUkIAQB7SVV9sqq+d7vaj1bVn29b7+6ndPef7GI/68cfpAfvpaGutV9M8tLufnh3f3BbsaoeN4KJba+uqr9dWv/u3T3Q9j//HfT5k6r63+/D57jP1uKYALAWDtRfZgCAVaqqg7v73jUcwuOTXLd9sbv/OsnDt61XVSd5WndvmTg2AGAPMhMCANbQ8myJqjqhqjZX1T1VdXtV/dLo9qfj/e4xA+CZVfV1VfXvqurGqrqjqi6qqkct7ffM0fbpqvq/tjvOz1fVW6vqt6rqniQ/Oo793qq6u6purapfqaoHL+2vq+olVXVDVX22ql5ZVU+sqveM8V6y3H+7z7jiWKvqIVX1uSQHJfnLqvqr3fi5PaSqfrGq/nr8rH61qh462i6vqtcu9b24qjZW1bcm+dUkzxw/x7tXe7ylff2rqrq+qu6qqiuq6vHb/Yx+fPyM7q6qN1RVjbaDquq1VfWpqvpEVb102+yWqnpVku9O8itjXL+ydMjvXWl/ALC/EkIAwL7j9Ule392PTPLEJJeM+rPG+yHjkoX3JvnR8fqeJN+UxYyBX0mSqjomyRuT/HCSw5M8KskR2x3rtCRvTXJIkt9O8qUkP53ksUmemeS5SV6y3TYnJzkuyYlJ/k2SNyX5l0mOSvLUJC/cwedacazd/YXu3jbT4Wnd/cQd/mS+1quTfHOSY5M8aXy+fz/a/lWSH6mq51TVDyc5IcnLuvv6JD+e5L3j53jIbhwvVXVakp9L8s+TrEvyZ0l+d7tu35fk25P8oyQvyOJnliQvTnLqGO/Tk5y+bYPu/rdjX9suSXnpKvYHAPslIQQA7F2/P/4V++7xL+9v3Enf/5XkSVX12O7+XHdftZO+P5zkl7r74939uSQvT3LGuG/E85P8QXf/eXd/MYs/znu77d/b3b/f3V/u7r/r7vd391XdfW93fzLJf07yj7fb5v/p7nu6+7okH07yR+P4n0nyziQ7uqnkzsa628ZsgHOS/HR3b+3uzyb5hSRnJEl335bkJ5JcmEWwc+boc3/9eJL/u7uvH5ev/EKSY5dnQyR5dXffPS4leVcWoUOyCBBe3903d/ddWYQoq7Gj/QHAfkkIAQB71+ndfci2V752dsGys7P41/2PVtXVVfV9O+n7D5LcuLR+Yxb3ejpstN20raG7P5/k09ttf9PySlV9c1W9o6puG5do/EIWsyKW3b60/HcrrD88K9vZWO+LdUm+Icn7l8KdPxz1bf4gi8s8PtbdO70R5W54fJLXLx1za5LKV88yuW1p+fP5ys/kq/6bbLe8MzvaHwDsl4QQALCP6O4buvuFSb4xyWuSvLWqHpavncWQJH+TxR/F2zwuyb1ZBAO3JjlyW8O4V8Jjtj/cdusXJPlokqPH5SA/l8Uf2HvCzsZ6X3wqi9DjKUsBz6OWLu1IklcluT7J4VW1fJnISj/L1bopyY8th0rd/dDufs8qtv2q/yZZXMKy7P6MCwD2G0IIANhHVNW/rKp13f3lJHeP8peT3Dnev2mp++8m+emqekJVPTyLmQu/Ny4TeGuS76+q7xg3i/z57DpQeESSe5J8rqqenMXlDHvKzsa628bP59eSvK6qvjFJquqIqjp5LD8ryYuSnJnkrCT/qaq2zVa4PcmRO7qJ5pKDq+rrl14PyuKmli+vqqeM4zyqqn5wlcO+JMnLxjgPSfKz27Xfnq/+7wsAByQhBADsO05Jct14YsTrk5wx7tfw+Sz+Zf/d41KAE5NsTPKbWTw54xNJ/j7JTybJuGfDTya5OIt/gf9ckjuSfGEnx/7XSX4oyWez+AP/9/bg59rhWO+Hn02yJclV4/KR/57kW6rqkUkuyuImj7d0958leXOS3xj3kvjjLB4HeltVfWon+78gi9kW216/0d1vy2KGysXjmB/O4maTq/FrSf4oyTVJPpjk8ixmg3xptL8+yfPHUzfOX+U+AWC/U91m/wHAgWzMPrg7i0stPrHGwyFJVZ2a5Fe7+/G77AwABxAzIQDgAFRV319V3zDuKfGLSa5N8sm1HdUDV1U9tKqeV1UHj0tDzkvytrUeFwDMJoQAgAPTaVncEPJvkhydxaUdpj+unUryH5LclcXlGNdn8ehUAHhAcTkGAAAAMIWZEAAAAMAUQggAAABgioPXegD31WMf+9hev379Wg8DAAAAWPL+97//U929bqW2/TaEWL9+fTZv3rzWwwAAAACWVNWNO2pzOQYAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBS7DCGq6luq6kNLr3uq6qeq6tFVtamqbhjvh47+VVXnV9WWqrqmqp6+tK+zRv8bquqspfpxVXXt2Ob8qqq983EBAACAtbLLEKK7P9bdx3b3sUmOS/L5JG9LsiHJld19dJIrx3qSnJrk6PE6J8kFSVJVj05yXpJnJDkhyXnbgovR58VL252yJz4cAAAAsO/Y3csxnpvkr7r7xiSnJblw1C9McvpYPi3JRb1wVZJDqurwJCcn2dTdW7v7riSbkpwy2h7Z3Vd1dye5aGlfAAAAwAFid0OIM5L87lg+rLtvHcu3JTlsLB+R5KalbW4etZ3Vb16hDgAAABxAVh1CVNWDk/yzJP9l+7Yxg6H34Lh2NIZzqmpzVW2+88479/bh9pr1Gy5b6yEAAADAdLszE+LUJB/o7tvH+u3jUoqM9ztG/ZYkRy1td+So7ax+5Ar1r9Hdb+ru47v7+HXr1u3G0AEAAIC1tjshxAvzlUsxkuTSJNuecHFWkrcv1c8cT8k4MclnxmUbVyQ5qaoOHTekPCnJFaPtnqo6cTwV48ylfR2wzIYAAADggebg1XSqqocl+SdJfmyp/Ookl1TV2UluTPKCUb88yfOSbMniSRovSpLu3lpVr0xy9ej3iu7eOpZfkuQtSR6a5J3jBQAAABxAVhVCdPffJnnMdrVPZ/G0jO37dpJzd7CfjUk2rlDfnOSpqxkLAAAAsH/a3adjAAAAANwnQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUqwohquqQqnprVX20qq6vqmdW1aOralNV3TDeDx19q6rOr6otVXVNVT19aT9njf43VNVZS/Xjqurasc35VVV7/qMCAAAAa2m1MyFen+QPu/vJSZ6W5PokG5Jc2d1HJ7lyrCfJqUmOHq9zklyQJFX16CTnJXlGkhOSnLctuBh9Xry03Sn372MBAAAA+5pdhhBV9agkz0ry5iTp7i92991JTkty4eh2YZLTx/JpSS7qhauSHFJVhyc5Ocmm7t7a3Xcl2ZTklNH2yO6+qrs7yUVL+wIAAAAOEKuZCfGEJHcm+Y2q+mBV/XpVPSzJYd196+hzW5LDxvIRSW5a2v7mUdtZ/eYV6gAAAMABZDUhxMFJnp7kgu7+tiR/m69cepEkGTMYes8P76tV1TlVtbmqNt955517+3AAAADAHrSaEOLmJDd39/vG+luzCCVuH5dSZLzfMdpvSXLU0vZHjtrO6keuUP8a3f2m7j6+u49ft27dKoYOAAAA7Ct2GUJ0921Jbqqqbxml5yb5SJJLk2x7wsVZSd4+li9NcuZ4SsaJST4zLtu4IslJVXXouCHlSUmuGG33VNWJ46kYZy7tCwAAADhAHLzKfj+Z5Ler6sFJPp7kRVkEGJdU1dlJbkzygtH38iTPS7IlyedH33T31qp6ZZKrR79XdPfWsfySJG9J8tAk7xwvAAAA4ACyqhCiuz+U5PgVmp67Qt9Ocu4O9rMxycYV6puTPHU1YwEAAAD2T6u5JwQAAADA/SaEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohxBpav+GytR4CAAAATCOEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQYrL1Gy5b6yEAAADAmhBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBSrCiGq6pNVdW1VfaiqNo/ao6tqU1XdMN4PHfWqqvOraktVXVNVT1/az1mj/w1VddZS/bix/y1j29rTHxQAAABYW7szE+J7uvvY7j5+rG9IcmV3H53kyrGeJKcmOXq8zklyQbIILZKcl+QZSU5Ict624GL0efHSdqfc508EAAAA7JPuz+UYpyW5cCxfmOT0pfpFvXBVkkOq6vAkJyfZ1N1bu/uuJJuSnDLaHtndV3V3J7loaV8AAADAAWK1IUQn+aOqen9VnTNqh3X3rWP5tiSHjeUjkty0tO3No7az+s0r1AEAAIADyMGr7Pdd3X1LVX1jkk1V9dHlxu7uquo9P7yvNgKQc5LkcY973N4+HAAAALAHrWomRHffMt7vSPK2LO7pcPu4lCLj/Y7R/ZYkRy1tfuSo7ax+5Ar1lcbxpu4+vruPX7du3WqGDgAAAOwjdhlCVNXDquoR25aTnJTkw0kuTbLtCRdnJXn7WL40yZnjKRknJvnMuGzjiiQnVdWh44aUJyW5YrTdU1UnjqdinLm0LwAAAOAAsZrLMQ5L8rbx1MyDk/xOd/9hVV2d5JKqOjvJjUleMPpfnuR5SbYk+XySFyVJd2+tqlcmuXr0e0V3bx3LL0nyliQPTfLO8QIAAAAOILsMIbr740metkL900meu0K9k5y7g31tTLJxhfrmJE9dxXgBAACA/dT9eUQnAAAAwKoJIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEKINbZ+w2VrPQQAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFKsOIarqoKr6YFW9Y6w/oareV1Vbqur3qurBo/6Qsb5ltK9f2sfLR/1jVXXyUv2UUdtSVRv24OcDAAAA9hG7MxPiZUmuX1p/TZLXdfeTktyV5OxRPzvJXaP+utEvVXVMkjOSPCXJKUneOIKNg5K8IcmpSY5J8sLRFwAAADiArCqEqKojk/zTJL8+1ivJc5K8dXS5MMnpY/m0sZ7R/tzR/7QkF3f3F7r7E0m2JDlhvLZ098e7+4tJLh59AQAAgAPIamdC/HKSf5Pky2P9MUnu7u57x/rNSY4Yy0ckuSlJRvtnRv//v77dNjuqAwAAAAeQXYYQVfV9Se7o7vdPGM+uxnJOVW2uqs133nnnWg8HAAAA2A2rmQnxnUn+WVV9MotLJZ6T5PVJDqmqg0efI5PcMpZvSXJUkoz2RyX59HJ9u212VP8a3f2m7j6+u49ft27dKoYOAAAA7Ct2GUJ098u7+8juXp/FjSX/uLt/OMm7kjx/dDsrydvH8qVjPaP9j7u7R/2M8fSMJyQ5OslfJLk6ydHjaRsPHse4dI98OgAAAGCfcfCuu+zQzya5uKr+Y5IPJnnzqL85yW9W1ZYkW7MIFdLd11XVJUk+kuTeJOd295eSpKpemuSKJAcl2djd192PcQEAAAD7oN0KIbr7T5L8yVj+eBZPtti+z98n+cEdbP+qJK9aoX55kst3ZywAAADA/mW1T8cAAAAAuF+EEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCl2GUJU1ddX1V9U1V9W1XVV9R9G/QlV9b6q2lJVv1dVDx71h4z1LaN9/dK+Xj7qH6uqk5fqp4zalqrasBc+JwAAALDGVjMT4gtJntPdT0tybJJTqurEJK9J8rruflKSu5KcPfqfneSuUX/d6JeqOibJGUmekuSUJG+sqoOq6qAkb0hyapJjkrxw9H3AWL/hsrUeAgAAAOx1uwwheuFzY/VB49VJnpPkraN+YZLTx/JpYz2j/blVVaN+cXd/obs/kWRLkhPGa0t3f7y7v5jk4tEXAAAAOICs6p4QY8bCh5LckWRTkr9Kcnd33zu63JzkiLF8RJKbkmS0fybJY5br222zozoAAABwAFlVCNHdX+ruY5McmcXMhSfvzUHtSFWdU1Wbq2rznXfeuRZDAAAAAO6j3Xo6RnffneRdSZ6Z5JCqOng0HZnklrF8S5KjkmS0PyrJp5fr222zo/pKx39Tdx/f3cevW7dud4YOAAAArLHVPB1jXVUdMpYfmuSfJLk+izDi+aPbWUnePpYvHesZ7X/c3T3qZ4ynZzwhydFJ/iLJ1UmOHk/beHAWN6+8dA98NgAAAGAfcvCuu+TwJBeOp1h8XZJLuvsdVfWRJBdX1X9M8sEkbx7935zkN6tqS5KtWYQK6e7rquqSJB9Jcm+Sc7v7S0lSVS9NckWSg5Js7O7r9tgnBAAAAPYJuwwhuvuaJN+2Qv3jWdwfYvv63yf5wR3s61VJXrVC/fIkl69ivAAAAMB+arfuCQEAAABwXwkhAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohxD5i/YbL1noIAAAAsFcJIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwxS5DiKo6qqreVVUfqarrquplo/7oqtpUVTeM90NHvarq/KraUlXXVNXTl/Z11uh/Q1WdtVQ/rqquHducX1W1Nz4sAAAAsHZWMxPi3iQ/093HJDkxyblVdUySDUmu7O6jk1w51pPk1CRHj9c5SS5IFqFFkvOSPCPJCUnO2xZcjD4vXtrulPv/0QAAAIB9yS5DiO6+tbs/MJY/m+T6JEckOS3JhaPbhUlOH8unJbmoF65KckhVHZ7k5CSbuntrd9+VZFOSU0bbI7v7qu7uJBct7QsAAAA4QOzWPSGqan2Sb0vyviSHdfeto+m2JIeN5SOS3LS02c2jtrP6zSvUAQAAgAPIqkOIqnp4kv+a5Ke6+57ltjGDoffw2FYawzlVtbmqNt955517+3AAAADAHrSqEKKqHpRFAPHb3f3fRvn2cSlFxvsdo35LkqOWNj9y1HZWP3KF+tfo7jd19/Hdffy6detWM3QAAABgH7Gap2NUkjcnub67f2mp6dIk255wcVaSty/VzxxPyTgxyWfGZRtXJDmpqg4dN6Q8KckVo+2eqjpxHOvMpX0BAAAAB4iDV9HnO5P8SJJrq+pDo/ZzSV6d5JKqOjvJjUleMNouT/K8JFuSfD7Ji5Kku7dW1SuTXD36vaK7t47llyR5S5KHJnnneAEAAAAHkF2GEN3950lqB83PXaF/Jzl3B/vamGTjCvXNSZ66q7EAAAAA+6/dejoGAAAAwH0lhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIAQAAAEwhhNiHrN9w2VoPAQAAAPYaIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCH2Mes3XLbWQwAAAIC9QggBAAAATLHLEKKqNlbVHVX14aXao6tqU1XdMN4PHfWqqvOraktVXVNVT1/a5qzR/4aqOmupflxVXTu2Ob+qak9/SAAAAGDtrWYmxFuSnLJdbUOSK7v76CRXjvUkOTXJ0eN1TpILkkVokeS8JM9IckKS87YFF6PPi5e22/5YAAAAwAFglyFEd/9pkq3blU9LcuFYvjDJ6Uv1i3rhqiSHVNXhSU5Osqm7t3b3XUk2JTlltD2yu6/q7k5y0dK+AAAAgAPIfb0nxGHdfetYvi3JYWP5iCQ3LfW7edR2Vr95hToAAABwgLnfN6YcMxh6D4xll6rqnKraXFWb77zzzhmHBAAAAPaQ+xpC3D4upch4v2PUb0ly1FK/I0dtZ/UjV6ivqLvf1N3Hd/fx69atu49DBwAAANbCfQ0hLk2y7QkXZyV5+1L9zPGUjBOTfGZctnFFkpOq6tBxQ8qTklwx2u6pqhPHUzHOXNoXAAAAcAA5eFcdqup3kzw7yWOr6uYsnnLx6iSXVNXZSW5M8oLR/fIkz0uyJcnnk7woSbp7a1W9MsnVo98runvbzS5fksUTOB6a5J3jBQAAABxgdhlCdPcLd9D03BX6dpJzd7CfjUk2rlDfnOSpuxoHAAAAsH+73zemBAAAAFgNIQQAAAAwhRACAAAAmEIIAQAAAEwhhAAAAACmEELsg9ZvuGythwAAAAB7nBACAAAAmEIIAQAAAEwhhAAAAACmEEIAAAAAUwghAAAAgCmEEAAAAMAUQggAAABgCiEEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAphBAAAADAFEIIAAAAYAohBAAAADCFEAIAAACYQgixj1q/4bK1HgIAAADsUUIIAAAAYAohBAAAADCFEAIAAACYQggBAAAATCGEAAAAAKYQQgAAAABTCCEAAACAKYQQ+7D1Gy5b6yEAAADAHiOEAAAAAKYQQgAAAABTCCEAAACAKYQQAAAAwBRCCAAAAGAKIQQAAAAwhRACAAAAmEIIsY9bv+GytR4CAAAA7BFCCAAAAGAKIQQAAAAwhRBiP+CSDAAAAA4EQggAAABgCiEEAAAAMIUQYj/hkgwAAAD2d0IIAAAAYAohxH7EbAgAAAD2Z0IIAAAAYIp9JoSoqlOq6mNVtaWqNqz1ePZVZkMAAACwv9onQoiqOijJG5KcmuSYJC+sqmPWdlT7LkEEAAAA+6OD13oAwwlJtnT3x5Okqi5OclqSj6zpqPZhy0HEJ1/9T9dwJAAAALA6+0oIcUSSm5bWb07yjDUay35nRzMjhBMAAADsS/aVEGJVquqcJOeM1c9V1cfWcjz3wWOTfGrWweo1s44Ee9XU8wYOAM4Z2H3OG9g9zhl25fE7athXQohbkhy1tH7kqH2V7n5TkjfNGtSeVlWbu/v4tR4H7E+cN7B7nDOw+5w3sHucM9wf+8SNKZNcneToqnpCVT04yRlJLl3jMQEAAAB70D4xE6K7762qlya5IslBSTZ293VrPCwAAABgD9onQogk6e7Lk1y+1uPYy/bbS0lgDTlvYPc4Z2D3OW9g9zhnuM+qu9d6DAAAAMADwL5yTwgAAADgACeEmKSqTqmqj1XVlqrasNbjgX1FVX2yqq6tqg9V1eZRe3RVbaqqG8b7oaNeVXX+OI+uqaqnr+3oYY6q2lhVd1TVh5dqu32eVNVZo/8NVXXWWnwWmGEH58zPV9Ut4/vmQ1X1vKW2l49z5mNVdfJS3e9vPCBU1VFV9a6q+khVXVdVLxt13zXscUKICarqoCRvSHJqkmOSvLCqjlnbUcE+5Xu6+9ilRz1tSHJldx+d5MqxnizOoaPH65wkF0wfKayNtyQ5Zbvabp0nVfXoJOcleUaSE5Kct+2XSTgAvSVfe84kyevG982x435kGb+TnZHkKWObN1bVQX5/4wHm3iQ/093HJDkxybnj/3ffNexxQog5Tkiypbs/3t1fTHJxktPWeEywLzstyYVj+cIkpy/VL+qFq5IcUlWHr8H4YKru/tMkW7cr7+55cnKSTd29tbvvSrIpK/+RBvu9HZwzO3Jakou7+wvd/YkkW7L43c3vbzxgdPet3f2BsfzZJNcnOSK+a9gLhBBzHJHkpqX1m0cNSDrJH1XV+6vqnFE7rLtvHcu3JTlsLDuX4Ct29zxx/kDy0jF1fOPSv846Z2BJVa1P8m1J3hffNewFQghgrX1Xdz89i2l951bVs5Ybe/EIH4/xgZ1wnsCqXJDkiUmOTXJrkteu6WhgH1RVD0/yX5P8VHffs9zmu4Y9RQgxxy1JjlpaP3LU4AGvu28Z73ckeVsW019v33aZxXi/Y3R3LsFX7O554vzhAa27b+/uL3X3l5P8WhbfN4lzBpIkVfWgLAKI3+7u/zbKvmvY44QQc1yd5OiqekJVPTiLmx9dusZjgjVXVQ+rqkdsW05yUpIPZ3F+bLub8llJ3j6WL01y5rgj84lJPrM0RRAeaHb3PLkiyUlVdeiYhn7SqMEDwnb3EPqBLL5vksU5c0ZVPaSqnpDFjfb+In5/4wGkqirJm5Nc392/tNTku4Y97uC1HsADQXffW1UvzeIEPCjJxu6+bo2HBfuCw5K8bfG9l4OT/E53/2FVXZ3kkqo6O8mNSV4w+l+e5HlZ3DTs80leNH/IMF9V/W6SZyd5bFXdnMWdx1+d3ThPuntrVb0yiz+skuQV3b3aG/fBfmUH58yzq+rYLKaTfzLJjyVJd19XVZck+UgWTwg4t7u/NPbj9zceKL4zyY8kubaqPjRqPxffNewFtbi0BwAAAGDvcjkGAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQDgAFJVr6uqn1pav6Kqfn1p/bVV9X/ex30/u6resQeGeV+OfVhVvaOq/rKqPlJVl+/l462vqg/vzWMAwAOREAIADizvTvIdSVJVX5fksUmestT+HUnes5odVdVBe3x0q1RVB29XekWSTd39tO4+JsmGNRgWAHA/CSEA4MDyniTPHMtPSfLhJJ+tqkOr6iFJvjXJB6rquVX1waq6tqo2jrZU1Ser6jVV9YEkP1hVp1TVR8f6P992kKr6x1X1ofH6YFU9YnkQYybBR6vqt6vq+qp6a1V9w2g7rqr+R1W9f8zUOHzU/6SqfrmqNid52Xaf6/AkN29b6e5rxjYPr6orq+oD47Octt3x31JV/3OM43ur6t1VdUNVnTD6/XxV/WZVvXfUX7z9D7SqDqqq/7eqrq6qa6rqx+7jfxsAeMATQgDAAaS7/ybJvVX1uCxmPbw3yfuyCCaOT3JtFt//b0nyL7r7HyY5OMlPLO3m09399CS/n+TXknx/kuOS/G9Lff51knO7+9gk353k71YYzrckeWN3f2uSe5K8pKoelOQ/JXl+dx+XZGOSVy1t8+DuPr67X7vdvt6Q5M1V9a6q+rdV9Q9G/e+T/MAY7/ckeW1V1Wh7UpLXJnnyeP1Qku8aY/+5pX3/oyTPGT+jf7+0723OTvKZ7v72JN+e5MVV9YQVPi8AsAtCCAA48LwniwBiWwjx3qX1d2cRDnyiu//n6H9hkmctbf974/3Jo98N3d1Jfmupz7uT/FJV/R9JDunue1cYx03d/e6x/FtZBADfkuSpSTZV1YeS/LskR65w7K/S3Vck+aYsQpEnJ/lgVa1LUkl+oaquSfLfkxyR5LCx2Se6+9ru/nKS65JcOT7HtUnWL+3+7d39d939qSTvSnLCdoc/KcmZY7zvS/KYJEevNE4AYOe2v94SANj/bbsvxD/M4nKMm5L8TBazEX5jFdv/7a46dPerq+qyJM9L8u6qOrm7P7p9txXWK8l13f3MrGyHx+7urUl+J8nvjBtkPivJI5KsS3Jcd/+vqvpkkq8fm3xhafMvL61/OV/9O9BK41xWSX5yBCEAwP1gJgQAHHjek+T7kmzt7i+NP94PyeJyg/ck+ViS9VX1pNH/R5L8jxX289HR74lj/YXbGqrqiWOWwWuSXJ3F7ITtPa6qtoUNP5Tkz8ex122rV9WDquopK2z7VarqOUv3lHhEkicm+eskj0pyxwggvifJ43e1rxWcVlVfX1WPSfLs8XmWXZHkJ8alJKmqb66qh92H4wDAA56ZEABw4Lk2i6di/M52tYePSw5SVS9K8l/GUyiuTvKr2++ku/++qs5JcllVfT7Jn2Ux8yBJfmr80b/tUod3rjCOjyU5t6o2JvlIkgu6+4tV9fwk51fVo7L4XeSXxz525rgkv1JV92bxjyi/3t1XV9UnkvxBVV2bZHMWwcnuuiaLyzAem+SV3f03VbV+qf3Xs7h84wPjfhN3Jjn9PhwHAB7wanFpJADAnjP+iH9Hdz91rceyM1X180k+192/uNZjAYAHApdjAAAAAFOYCQEAAABMYSYEAAAAMIUQAgAAAJhCCAEAAABMIYQAAAAAphBCAAAAAFMIIQAAAIAp/j+op2QHHoyvLwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "train_data['input_len'] = train_data['text'].str.split().str.len()\n",
        "eval_data['input_len'] = eval_data['text'].str.split().str.len()\n",
        "\n",
        "max_len = max(max(train_data['input_len']), max(eval_data['input_len']))\n",
        "print(f\"The maximum length of this dataset is approximately {max_len} words.\")\n",
        "\n",
        "input_lengths = pd.DataFrame()\n",
        "input_lengths = pd.concat([train_data['input_len'], eval_data['input_len']])\n",
        "\n",
        "n = max_len\n",
        "input_lengths.hist(grid=False, figsize=(18,8), range=(0, n), bins=n)\n",
        "plt.suptitle('')\n",
        "plt.title('Histogram of Text Length')\n",
        "plt.ylabel('')\n",
        "plt.xlabel('Words per Sample')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C6nSgL70QI8f"
      },
      "source": [
        "#### Remove Overly Lengthy Inputs & Unnecessary Features"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "sGs2Mw-kQI8f",
        "outputId": "0f0f629d-88ca-43f4-c3d3-c9eba96e43a9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 676493 entries, 0 to 707540\n",
            "Data columns (total 1 columns):\n",
            " #   Column  Non-Null Count   Dtype \n",
            "---  ------  --------------   ----- \n",
            " 0   text    676493 non-null  object\n",
            "dtypes: object(1)\n",
            "memory usage: 10.3+ MB\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 292916 entries, 0 to 303231\n",
            "Data columns (total 1 columns):\n",
            " #   Column  Non-Null Count   Dtype \n",
            "---  ------  --------------   ----- \n",
            " 0   text    292916 non-null  object\n",
            "dtypes: object(1)\n",
            "memory usage: 4.5+ MB\n"
          ]
        }
      ],
      "source": [
        "train_data = train_data[train_data[\"input_len\"] < 250]\n",
        "eval_data = eval_data[eval_data[\"input_len\"] < 250]\n",
        "\n",
        "\n",
        "train_data = train_data.drop(columns=[\"input_len\"])\n",
        "eval_data = eval_data.drop(columns=[\"input_len\"])\n",
        "\n",
        "train_data.info()\n",
        "eval_data.info()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DIqLyPnFQI8g"
      },
      "source": [
        "#### Convert DataFrame to Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "wehkHCELQI8g"
      },
      "outputs": [],
      "source": [
        "training_dataset = Dataset.from_pandas(train_data)\n",
        "evaluation_dataset = Dataset.from_pandas(eval_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iTCnSi9RQI8g"
      },
      "source": [
        "#### Split Dataset into Training & Evaluation Datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "WTA2uMBlQI8g",
        "outputId": "a372c276-ed39-4d8d-88e6-a3715bcefbf2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training data shape: (676493, 2)\n",
            "Eval data shape: (292916, 2)\n"
          ]
        }
      ],
      "source": [
        "ds = DatasetDict({\n",
        "    'train' : training_dataset,\n",
        "    'eval' : evaluation_dataset\n",
        "})\n",
        "\n",
        "print('Training data shape:', ds['train'].shape)\n",
        "print('Eval data shape:', ds['eval'].shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xgphUi1hQI8g"
      },
      "source": [
        "#### Basic Values/Constants"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "K1hdlJtAQI8g"
      },
      "outputs": [],
      "source": [
        "BLOCK_SIZE = 128\n",
        "MODEL_CKPT = \"bert-base-uncased\"\n",
        "\n",
        "MODEL_NAME = MODEL_CKPT.split(\"/\")[-1] + \"-Masked_Language_Modeling-Reddit_Comments\"\n",
        "set_seed = 42\n",
        "\n",
        "BATCH_SIZE = 1000\n",
        "NUM_OF_EPOCHS = 3\n",
        "\n",
        "WEIGHT_DECAY = 0.01\n",
        "STRATEGY = \"epoch\"\n",
        "\n",
        "REPORTS_TO = \"tensorboard\"\n",
        "LEARNING_RATE = 2e-5\n",
        "\n",
        "DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5hRw9dWWQI8g"
      },
      "source": [
        "#### Tokenize Entire Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "qFV8-OU5QI8g"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(MODEL_CKPT, use_fast=True)\n",
        "\n",
        "def tokenizer_function(samples):\n",
        "    return tokenizer(samples[\"text\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "46aK7FoHQI8g"
      },
      "source": [
        "#### Apply Tokenizer Function to Entire Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "d7380e61b15646859f3447c62ba0c15c",
            "6f8c29e0b9544a889d279c777336d50a",
            "d88f39b5b2d24972a94cb1e2dd1b8751",
            "ca9c19ed8a8f462f8dce9de58558b73c",
            "f725aa904b3449faa3da1ca427e7a251",
            "619d7635c886465993c829bcbe783a5b",
            "5995d93c7ffa43baa098e99f260846c5",
            "3ffd307f28124a3f8240ea008315add7",
            "f90ffc33242d4eb7ac48e05056235310",
            "0ee0757c99be42ecaafa324998d7a247",
            "307671afe7cd4c118e9ca6f39b00e3ef",
            "99694d118c0f4ac0a57612dce5480ffb",
            "1762acebf1db4c4ebba469e3520eece6",
            "a98a6946098c427ab5c720cc768954cc",
            "0a80cea7f3874ddd9ecf71d5fe9c24a7",
            "004fb7a1107b4f808c76dcf94a4fe584",
            "1f9f39b791a14005bc9932b9d0201d67",
            "3045c8bcbd9b4d69968ee72cdd72771c",
            "61f1bca19093414797b05426d576eee8",
            "eeef52343bfb4e489c5d533cc4b51b11",
            "c2716ee5c0be4e9eae0492ecc78b0cfb",
            "4384d46f01854aefbf6d9099e7635a5e"
          ],
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "id": "6KTUWSpNQI8g",
        "outputId": "075ced88-e1f9-4d52-dd8f-7a16d6cc4e5f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/676493 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d7380e61b15646859f3447c62ba0c15c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (620 > 512). Running this sequence through the model will result in indexing errors\n",
            "Token indices sequence length is longer than the specified maximum sequence length for this model (580 > 512). Running this sequence through the model will result in indexing errors\n",
            "Token indices sequence length is longer than the specified maximum sequence length for this model (571 > 512). Running this sequence through the model will result in indexing errors\n",
            "Token indices sequence length is longer than the specified maximum sequence length for this model (919 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/292916 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "99694d118c0f4ac0a57612dce5480ffb"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (680 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset({\n",
            "    features: ['input_ids', 'token_type_ids', 'attention_mask'],\n",
            "    num_rows: 676493\n",
            "})\n",
            "Dataset({\n",
            "    features: ['input_ids', 'token_type_ids', 'attention_mask'],\n",
            "    num_rows: 292916\n",
            "})\n"
          ]
        }
      ],
      "source": [
        "tokenized_ds = ds.map(tokenizer_function, \n",
        "                      batched=True,\n",
        "                      num_proc=4,\n",
        "                      remove_columns=ds[\"train\"].column_names,)\n",
        "\n",
        "print(tokenized_ds[\"train\"])\n",
        "print(tokenized_ds[\"eval\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6tW01iTPQI8g"
      },
      "source": [
        "#### Preprocess Tokenized Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "a98Ly1PQQI8g"
      },
      "outputs": [],
      "source": [
        "def group_texts(samples):\n",
        "    concatenated_samples = {k: sum(samples[k], []) for k in samples.keys()}\n",
        "    total_length = len(concatenated_samples[list(samples.keys())[0]])\n",
        "    \n",
        "    if total_length >= BLOCK_SIZE:\n",
        "        total_length = (total_length // BLOCK_SIZE) * BLOCK_SIZE\n",
        "    \n",
        "    result = {\n",
        "        k: [t[i : i + BLOCK_SIZE] for i in range(0, total_length, BLOCK_SIZE)]\n",
        "        for k, t in concatenated_samples.items()\n",
        "    }\n",
        "    result[\"labels\"] = result[\"input_ids\"].copy()\n",
        "    return result"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g5LwQ9IlQI8h"
      },
      "source": [
        "#### Apply Preprocessing Function to Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "referenced_widgets": [
            "6e2caa18b6a24bc3ac3705283863af2f",
            "80da1988470b4e93963c98939b02a44b",
            "a7140cb1d88a4fa38e9b51271e2c892b",
            "3e8105242d5243df997178a1cdb38767",
            "9172f3cb747f41d881c0ca0e7ad62bc6",
            "1f0adc70a63649e7a53666df2ed55e23",
            "02c00646e36941ada4e2b9498191c206",
            "378f5d5bf3664e91b069f4514238114f",
            "dff262c97eac4acc95a151481881cfd4",
            "1b3e4a024f4c402cbfe74b1737a3d330",
            "777717b1d7554da58314c567a0954d46",
            "5919fa88725245d7aae90a338b0b48d6",
            "efe8d287a5014043bd04474a2c1a05ff",
            "2dea5214fcd041fe8f2c9f8bf9d5abc4",
            "a3a51e9b99cc485b93e3daa7810fcca9",
            "85c2f5b0730444e3aab13ca71c3dccc2",
            "ef9809a52bca478f8dc9356e75abab88",
            "7937d26587c7468485d02e2a87cc88bc",
            "f35f913c639745bf9943dedbf2f2d013",
            "721c4881dfc8427e82f8ccd041cfde84",
            "4eb2b70fd3cc4d3380dbe6a03c467b4d",
            "8980b527500f4c58aefefc04e75ac7b7"
          ],
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "DjDBBeM1QI8h",
        "outputId": "765ceb3e-02bb-4498-bd6e-71b1f1ca0143"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/676493 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6e2caa18b6a24bc3ac3705283863af2f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map (num_proc=4):   0%|          | 0/292916 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5919fa88725245d7aae90a338b0b48d6"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "mlm_ds = tokenized_ds.map(\n",
        "    group_texts,\n",
        "    batched=True,\n",
        "    num_proc=4,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EyZH87qvQI8h"
      },
      "source": [
        "#### Instantiate Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "dskKcH8wQI8h",
        "outputId": "d4295723-64a6-4f78-c992-3045b46baa7a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121,
          "referenced_widgets": [
            "f712bad0177746c5ba7f94d3e5b6cbd3",
            "8ce0eb37c2924a449f65b3a380158bd4",
            "ba78b9ac0068447b979af31a6d2a30f7",
            "7075ece6a473478eb3550c55a8153076",
            "faefe4cda4ee46dea5ef8f25217e5954",
            "cb8964f3f8e743dfa4c6ee823f848ec6",
            "31aa614f6fcc419cae06906e735ac4fa",
            "73a3c6e3ac6c49428addac527d5ad6d3",
            "c88334dab2e4453299e682429d6b6730",
            "a10ba366015d4e2eae00002e424179ce",
            "e01d95495cb64029862ee2ad9a866a96"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f712bad0177746c5ba7f94d3e5b6cbd3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "model = (\n",
        "    AutoModelForMaskedLM.from_pretrained(MODEL_CKPT)\n",
        "    ).to(DEVICE)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kkCdnxiXQI8h"
      },
      "source": [
        "#### Define TrainingArguments"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "55wGfgJFQI8h"
      },
      "outputs": [],
      "source": [
        "args = TrainingArguments(\n",
        "    output_dir=MODEL_NAME,\n",
        "    evaluation_strategy=STRATEGY,\n",
        "    learning_rate=LEARNING_RATE,\n",
        "    weight_decay=WEIGHT_DECAY,\n",
        "    num_train_epochs=NUM_OF_EPOCHS,\n",
        "    report_to=REPORTS_TO,\n",
        "    logging_first_step=True,\n",
        "    logging_strategy=STRATEGY,\n",
        "    save_strategy=STRATEGY,\n",
        "    hub_private_repo=True,\n",
        "    push_to_hub=True\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VQXXw9lcQI8h"
      },
      "source": [
        "#### Define Data Collator"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "7fjw6NFJQI8h"
      },
      "outputs": [],
      "source": [
        "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E91UJqZEQI8h"
      },
      "source": [
        "#### Instantiate Trainer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "IMxb8OxCQI8h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "714d617d-f444-4573-b873-36e47b68ab7e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Cloning https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments into local empty directory.\n",
            "WARNING:huggingface_hub.repository:Cloning https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments into local empty directory.\n"
          ]
        }
      ],
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=args,\n",
        "    train_dataset=mlm_ds[\"train\"],\n",
        "    eval_dataset=mlm_ds[\"eval\"],\n",
        "    data_collator=data_collator,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h8aao3DFQI8h"
      },
      "source": [
        "#### Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 242
        },
        "id": "Jwt-kWiaQI8h",
        "outputId": "2327228a-8eec-4912-c359-48ac40456bb2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='32436' max='32436' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [32436/32436 2:35:25, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.875700</td>\n",
              "      <td>2.638240</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.681800</td>\n",
              "      <td>2.569892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2.610300</td>\n",
              "      <td>2.540241</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "train_results = trainer.train()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iYA2a7fzQI8h"
      },
      "source": [
        "#### Evaluate Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "oA29Q3L_QI8h",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "72bbda83-f63b-4d51-bb1f-ef71791ec98c"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='4642' max='4642' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [4642/4642 05:33]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Perplexity: 12.70\n"
          ]
        }
      ],
      "source": [
        "evaluation_results = trainer.evaluate()\n",
        "print(f\"Perplexity: {math.exp(evaluation_results['eval_loss']):.2f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jmicj1FEQI8h"
      },
      "source": [
        "#### Push Model to HuggingFace Hub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "q51x7lqUQI8h",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526,
          "referenced_widgets": [
            "2dab3ad8275c49c38653c31c948ff84a",
            "096bfeb14b224ded9726ac0e9f1ebb65",
            "9126cb828c194029b464bbcb44815ef5",
            "5cb47bfc64ca48d89d45c83e8ad342dd",
            "312732696e2d4881ba04976ff4fd185e",
            "c4e193e40c58426aa57a9b01e99d7ee8",
            "b3f921912cc8421ca7b33c12917e4c9f",
            "e6e15c6e132045d1babac6ed1bb9fac8",
            "a61d170273674b4a8791469a61d7ad05",
            "c3e4fe2398d6403889cdff9a396d0335",
            "ab535b4c034146539a1618003b607827",
            "10067f1848d84004ace7875f76956171",
            "9bc670406d924bd48260985fff7e5fb3",
            "8996b222438e420fa075ddcac328846c",
            "2661ce702cd246dc9c43ef1dd6747a12",
            "15bc33847c7242dfb1d9adc3e772f26f",
            "4a8d20cffffc4e0883d5c41a981b6100",
            "574e1fdc1bda4638bb62cbd2f1c27796",
            "e8d2ea95edd949b1ac778db0a21016ae",
            "c6a825650ded4ccdb349df57a77ba3a5",
            "5b4076635bb34264a25bf5763c0a5e38",
            "8df2874a570b4f11b6f69decca52e7ca"
          ]
        },
        "outputId": "251fbc2f-d30d-48a0-ca55-89d0ad99309e"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Upload file runs/Mar15_17-25-41_8cb5130871e0/events.out.tfevents.1678901147.8cb5130871e0.163.0: 100%|#########…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2dab3ad8275c49c38653c31c948ff84a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Upload file runs/Mar15_17-25-41_8cb5130871e0/events.out.tfevents.1678910810.8cb5130871e0.163.2: 100%|#########…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "10067f1848d84004ace7875f76956171"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "remote: Scanning LFS files of refs/heads/main for validity...        \n",
            "remote: LFS file scan complete.        \n",
            "To https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments\n",
            "   a5a14a0..fe0600b  main -> main\n",
            "\n",
            "WARNING:huggingface_hub.repository:remote: Scanning LFS files of refs/heads/main for validity...        \n",
            "remote: LFS file scan complete.        \n",
            "To https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments\n",
            "   a5a14a0..fe0600b  main -> main\n",
            "\n",
            "To https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments\n",
            "   fe0600b..7960a45  main -> main\n",
            "\n",
            "WARNING:huggingface_hub.repository:To https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments\n",
            "   fe0600b..7960a45  main -> main\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'https://huggingface.co/DunnBC22/bert-base-uncased-Masked_Language_Modeling-Reddit_Comments/commit/fe0600bae9eb99ba2b2f6118f8a29366701df06a'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "trainer.push_to_hub()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0UOt8W7QQI8h"
      },
      "source": [
        "### Citations\n",
        "\n",
        "- Model Checkpoint\n",
        "> @article{DBLP:journals/corr/abs-1810-04805, author ={Jacob Devlin and Ming{-}Wei Chang and Kenton Lee and Kristina Toutanova}, title = {{BERT:} Pre-training of Deep Bidirectional Transformers for Language Understanding},\n",
        "  journal   = {CoRR},\n",
        "  volume    = {abs/1810.04805},\n",
        "  year      = {2018},\n",
        "  url       = {http://arxiv.org/abs/1810.04805},\n",
        "  archivePrefix = {arXiv},\n",
        "  eprint    = {1810.04805},\n",
        "  timestamp = {Tue, 30 Oct 2018 20:39:56 +0100},\n",
        "  biburl    = {https://dblp.org/rec/journals/corr/abs-1810-04805.bib},\n",
        "  bibsource = {dblp computer science bibliography, https://dblp.org}\n",
        "}\n",
        "\n",
        "- Metric (Perplexity)\n",
        "> @article{jelinek1977perplexity,\n",
        "title={Perplexity—a measure of the difficulty of speech recognition tasks},\n",
        "author={Jelinek, Fred and Mercer, Robert L and Bahl, Lalit R and Baker, James K},\n",
        "journal={The Journal of the Acoustical Society of America},\n",
        "volume={62},\n",
        "number={S1},\n",
        "pages={S63--S63},\n",
        "year={1977},\n",
        "publisher={Acoustical Society of America}\n",
        "}\n",
        "\n"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "nlp_ml",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "0b7a754c5aaaa44a21e2f2148f50b931a5abdffe7ef71537f5954179554f39df"
      }
    },
    "colab": {
      "provenance": []
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "d7380e61b15646859f3447c62ba0c15c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6f8c29e0b9544a889d279c777336d50a",
              "IPY_MODEL_d88f39b5b2d24972a94cb1e2dd1b8751",
              "IPY_MODEL_ca9c19ed8a8f462f8dce9de58558b73c"
            ],
            "layout": "IPY_MODEL_f725aa904b3449faa3da1ca427e7a251"
          }
        },
        "6f8c29e0b9544a889d279c777336d50a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_619d7635c886465993c829bcbe783a5b",
            "placeholder": "​",
            "style": "IPY_MODEL_5995d93c7ffa43baa098e99f260846c5",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "d88f39b5b2d24972a94cb1e2dd1b8751": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3ffd307f28124a3f8240ea008315add7",
            "max": 676493,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f90ffc33242d4eb7ac48e05056235310",
            "value": 676493
          }
        },
        "ca9c19ed8a8f462f8dce9de58558b73c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ee0757c99be42ecaafa324998d7a247",
            "placeholder": "​",
            "style": "IPY_MODEL_307671afe7cd4c118e9ca6f39b00e3ef",
            "value": " 675370/676493 [00:58&lt;00:00, 6452.39 examples/s]"
          }
        },
        "f725aa904b3449faa3da1ca427e7a251": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "619d7635c886465993c829bcbe783a5b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5995d93c7ffa43baa098e99f260846c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3ffd307f28124a3f8240ea008315add7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f90ffc33242d4eb7ac48e05056235310": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0ee0757c99be42ecaafa324998d7a247": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "307671afe7cd4c118e9ca6f39b00e3ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "99694d118c0f4ac0a57612dce5480ffb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1762acebf1db4c4ebba469e3520eece6",
              "IPY_MODEL_a98a6946098c427ab5c720cc768954cc",
              "IPY_MODEL_0a80cea7f3874ddd9ecf71d5fe9c24a7"
            ],
            "layout": "IPY_MODEL_004fb7a1107b4f808c76dcf94a4fe584"
          }
        },
        "1762acebf1db4c4ebba469e3520eece6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f9f39b791a14005bc9932b9d0201d67",
            "placeholder": "​",
            "style": "IPY_MODEL_3045c8bcbd9b4d69968ee72cdd72771c",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "a98a6946098c427ab5c720cc768954cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_61f1bca19093414797b05426d576eee8",
            "max": 292916,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eeef52343bfb4e489c5d533cc4b51b11",
            "value": 292916
          }
        },
        "0a80cea7f3874ddd9ecf71d5fe9c24a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2716ee5c0be4e9eae0492ecc78b0cfb",
            "placeholder": "​",
            "style": "IPY_MODEL_4384d46f01854aefbf6d9099e7635a5e",
            "value": " 291458/292916 [00:23&lt;00:00, 11172.16 examples/s]"
          }
        },
        "004fb7a1107b4f808c76dcf94a4fe584": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "1f9f39b791a14005bc9932b9d0201d67": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3045c8bcbd9b4d69968ee72cdd72771c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "61f1bca19093414797b05426d576eee8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eeef52343bfb4e489c5d533cc4b51b11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c2716ee5c0be4e9eae0492ecc78b0cfb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4384d46f01854aefbf6d9099e7635a5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6e2caa18b6a24bc3ac3705283863af2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_80da1988470b4e93963c98939b02a44b",
              "IPY_MODEL_a7140cb1d88a4fa38e9b51271e2c892b",
              "IPY_MODEL_3e8105242d5243df997178a1cdb38767"
            ],
            "layout": "IPY_MODEL_9172f3cb747f41d881c0ca0e7ad62bc6"
          }
        },
        "80da1988470b4e93963c98939b02a44b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f0adc70a63649e7a53666df2ed55e23",
            "placeholder": "​",
            "style": "IPY_MODEL_02c00646e36941ada4e2b9498191c206",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "a7140cb1d88a4fa38e9b51271e2c892b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_378f5d5bf3664e91b069f4514238114f",
            "max": 676493,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dff262c97eac4acc95a151481881cfd4",
            "value": 676493
          }
        },
        "3e8105242d5243df997178a1cdb38767": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b3e4a024f4c402cbfe74b1737a3d330",
            "placeholder": "​",
            "style": "IPY_MODEL_777717b1d7554da58314c567a0954d46",
            "value": " 675370/676493 [01:10&lt;00:00, 3549.85 examples/s]"
          }
        },
        "9172f3cb747f41d881c0ca0e7ad62bc6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "1f0adc70a63649e7a53666df2ed55e23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02c00646e36941ada4e2b9498191c206": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "378f5d5bf3664e91b069f4514238114f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dff262c97eac4acc95a151481881cfd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1b3e4a024f4c402cbfe74b1737a3d330": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "777717b1d7554da58314c567a0954d46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5919fa88725245d7aae90a338b0b48d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_efe8d287a5014043bd04474a2c1a05ff",
              "IPY_MODEL_2dea5214fcd041fe8f2c9f8bf9d5abc4",
              "IPY_MODEL_a3a51e9b99cc485b93e3daa7810fcca9"
            ],
            "layout": "IPY_MODEL_85c2f5b0730444e3aab13ca71c3dccc2"
          }
        },
        "efe8d287a5014043bd04474a2c1a05ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef9809a52bca478f8dc9356e75abab88",
            "placeholder": "​",
            "style": "IPY_MODEL_7937d26587c7468485d02e2a87cc88bc",
            "value": "Map (num_proc=4): 100%"
          }
        },
        "2dea5214fcd041fe8f2c9f8bf9d5abc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f35f913c639745bf9943dedbf2f2d013",
            "max": 292916,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_721c4881dfc8427e82f8ccd041cfde84",
            "value": 292916
          }
        },
        "a3a51e9b99cc485b93e3daa7810fcca9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4eb2b70fd3cc4d3380dbe6a03c467b4d",
            "placeholder": "​",
            "style": "IPY_MODEL_8980b527500f4c58aefefc04e75ac7b7",
            "value": " 292687/292916 [00:31&lt;00:00, 6621.62 examples/s]"
          }
        },
        "85c2f5b0730444e3aab13ca71c3dccc2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "ef9809a52bca478f8dc9356e75abab88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7937d26587c7468485d02e2a87cc88bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f35f913c639745bf9943dedbf2f2d013": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "721c4881dfc8427e82f8ccd041cfde84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4eb2b70fd3cc4d3380dbe6a03c467b4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8980b527500f4c58aefefc04e75ac7b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f712bad0177746c5ba7f94d3e5b6cbd3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8ce0eb37c2924a449f65b3a380158bd4",
              "IPY_MODEL_ba78b9ac0068447b979af31a6d2a30f7",
              "IPY_MODEL_7075ece6a473478eb3550c55a8153076"
            ],
            "layout": "IPY_MODEL_faefe4cda4ee46dea5ef8f25217e5954"
          }
        },
        "8ce0eb37c2924a449f65b3a380158bd4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb8964f3f8e743dfa4c6ee823f848ec6",
            "placeholder": "​",
            "style": "IPY_MODEL_31aa614f6fcc419cae06906e735ac4fa",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "ba78b9ac0068447b979af31a6d2a30f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73a3c6e3ac6c49428addac527d5ad6d3",
            "max": 440473133,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c88334dab2e4453299e682429d6b6730",
            "value": 440473133
          }
        },
        "7075ece6a473478eb3550c55a8153076": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a10ba366015d4e2eae00002e424179ce",
            "placeholder": "​",
            "style": "IPY_MODEL_e01d95495cb64029862ee2ad9a866a96",
            "value": " 440M/440M [00:01&lt;00:00, 242MB/s]"
          }
        },
        "faefe4cda4ee46dea5ef8f25217e5954": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb8964f3f8e743dfa4c6ee823f848ec6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31aa614f6fcc419cae06906e735ac4fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73a3c6e3ac6c49428addac527d5ad6d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c88334dab2e4453299e682429d6b6730": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a10ba366015d4e2eae00002e424179ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e01d95495cb64029862ee2ad9a866a96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2dab3ad8275c49c38653c31c948ff84a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_096bfeb14b224ded9726ac0e9f1ebb65",
              "IPY_MODEL_9126cb828c194029b464bbcb44815ef5",
              "IPY_MODEL_5cb47bfc64ca48d89d45c83e8ad342dd"
            ],
            "layout": "IPY_MODEL_312732696e2d4881ba04976ff4fd185e"
          }
        },
        "096bfeb14b224ded9726ac0e9f1ebb65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c4e193e40c58426aa57a9b01e99d7ee8",
            "placeholder": "​",
            "style": "IPY_MODEL_b3f921912cc8421ca7b33c12917e4c9f",
            "value": "Upload file runs/Mar15_17-25-41_8cb5130871e0/events.out.tfevents.1678901147.8cb5130871e0.163.0: 100%"
          }
        },
        "9126cb828c194029b464bbcb44815ef5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e6e15c6e132045d1babac6ed1bb9fac8",
            "max": 5842,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a61d170273674b4a8791469a61d7ad05",
            "value": 5842
          }
        },
        "5cb47bfc64ca48d89d45c83e8ad342dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3e4fe2398d6403889cdff9a396d0335",
            "placeholder": "​",
            "style": "IPY_MODEL_ab535b4c034146539a1618003b607827",
            "value": " 5.71k/5.71k [00:01&lt;?, ?B/s]"
          }
        },
        "312732696e2d4881ba04976ff4fd185e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4e193e40c58426aa57a9b01e99d7ee8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3f921912cc8421ca7b33c12917e4c9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e6e15c6e132045d1babac6ed1bb9fac8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a61d170273674b4a8791469a61d7ad05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c3e4fe2398d6403889cdff9a396d0335": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ab535b4c034146539a1618003b607827": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "10067f1848d84004ace7875f76956171": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9bc670406d924bd48260985fff7e5fb3",
              "IPY_MODEL_8996b222438e420fa075ddcac328846c",
              "IPY_MODEL_2661ce702cd246dc9c43ef1dd6747a12"
            ],
            "layout": "IPY_MODEL_15bc33847c7242dfb1d9adc3e772f26f"
          }
        },
        "9bc670406d924bd48260985fff7e5fb3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a8d20cffffc4e0883d5c41a981b6100",
            "placeholder": "​",
            "style": "IPY_MODEL_574e1fdc1bda4638bb62cbd2f1c27796",
            "value": "Upload file runs/Mar15_17-25-41_8cb5130871e0/events.out.tfevents.1678910810.8cb5130871e0.163.2: 100%"
          }
        },
        "8996b222438e420fa075ddcac328846c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e8d2ea95edd949b1ac778db0a21016ae",
            "max": 316,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c6a825650ded4ccdb349df57a77ba3a5",
            "value": 316
          }
        },
        "2661ce702cd246dc9c43ef1dd6747a12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b4076635bb34264a25bf5763c0a5e38",
            "placeholder": "​",
            "style": "IPY_MODEL_8df2874a570b4f11b6f69decca52e7ca",
            "value": " 316/316 [00:01&lt;?, ?B/s]"
          }
        },
        "15bc33847c7242dfb1d9adc3e772f26f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a8d20cffffc4e0883d5c41a981b6100": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "574e1fdc1bda4638bb62cbd2f1c27796": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e8d2ea95edd949b1ac778db0a21016ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6a825650ded4ccdb349df57a77ba3a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5b4076635bb34264a25bf5763c0a5e38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8df2874a570b4f11b6f69decca52e7ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}