[Nemat_Aloush_J41332c]HW4_tune.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nXpkXz1QJmhJ"
      },
      "source": [
        "Soft deadline: `30.03.2022 23:59`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QGbs18XJJiJa"
      },
      "source": [
        "Student: Nemat Allah Aloush\n",
        "\n",
        "Group number : J41332c\n",
        "\n",
        "ISU number : 336092\n",
        "\n",
        "In case GitHub is not renderein the output cells, https://nbviewer.org/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yxy7euTpCbGp"
      },
      "source": [
        "In this homework you will understand the fine-tuning procedure and get acquainted with Huggingface Datasets library"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7D86UDkvFVf7",
        "outputId": "fab72643-c58e-42ce-c04b-dd86534c06a2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "# Mounting to google drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive/')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_1MXcMymXeCx",
        "outputId": "abb7b1e0-2ce5-4ede-e4f7-dc4337d12c67"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in c:\\users\\user\\anaconda3\\lib\\site-packages (4.16.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (0.4.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (2021.8.3)\n",
            "Requirement already satisfied: requests in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (2.26.0)\n",
            "Requirement already satisfied: filelock in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (3.3.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,>=0.10.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: sacremoses in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (0.0.49)\n",
            "Requirement already satisfied: packaging>=20.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (21.0)\n",
            "Requirement already satisfied: numpy>=1.17 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (1.20.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in c:\\users\\user\\anaconda3\\lib\\site-packages (from transformers) (4.62.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\user\\anaconda3\\lib\\site-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\user\\anaconda3\\lib\\site-packages (from packaging>=20.0->transformers) (3.0.4)\n",
            "Requirement already satisfied: colorama in c:\\users\\user\\anaconda3\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests->transformers) (1.26.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests->transformers) (3.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests->transformers) (2.0.4)\n",
            "Requirement already satisfied: six in c:\\users\\user\\anaconda3\\lib\\site-packages (from sacremoses->transformers) (1.16.0)\n",
            "Requirement already satisfied: click in c:\\users\\user\\anaconda3\\lib\\site-packages (from sacremoses->transformers) (8.0.3)\n",
            "Requirement already satisfied: joblib in c:\\users\\user\\anaconda3\\lib\\site-packages (from sacremoses->transformers) (1.1.0)\n"
          ]
        }
      ],
      "source": [
        "! pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zD6H1CPcMfNH",
        "outputId": "89676f2f-a305-40a2-89a8-a7662647bdb8"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/huggingface/datasets.git\n",
            "  Cloning https://github.com/huggingface/datasets.git to c:\\users\\user\\appdata\\local\\temp\\pip-req-build-oiifhv47\n",
            "  Resolved https://github.com/huggingface/datasets.git to commit bfb3d0916e95ae1d14412cbdbbc7d24cd91170c2\n",
            "Requirement already satisfied: numpy>=1.17 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (1.20.3)\n",
            "Requirement already satisfied: pyarrow>=5.0.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (7.0.0)\n",
            "Requirement already satisfied: dill in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (0.3.4)\n",
            "Requirement already satisfied: pandas in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (1.3.4)\n",
            "Requirement already satisfied: requests>=2.19.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (2.26.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (4.62.3)\n",
            "Requirement already satisfied: xxhash in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (3.0.0)\n",
            "Requirement already satisfied: multiprocess in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (0.70.12.2)\n",
            "Requirement already satisfied: fsspec[http]>=2021.05.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (2021.10.1)\n",
            "Requirement already satisfied: aiohttp in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (3.8.1)\n",
            "Requirement already satisfied: huggingface_hub<1.0.0,>=0.1.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (0.4.0)\n",
            "Requirement already satisfied: packaging in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (21.0)\n",
            "Requirement already satisfied: responses<0.19 in c:\\users\\user\\anaconda3\\lib\\site-packages (from datasets==2.0.1.dev0) (0.18.0)\n",
            "Requirement already satisfied: filelock in c:\\users\\user\\anaconda3\\lib\\site-packages (from huggingface_hub<1.0.0,>=0.1.0->datasets==2.0.1.dev0) (3.3.1)\n",
            "Requirement already satisfied: pyyaml in c:\\users\\user\\anaconda3\\lib\\site-packages (from huggingface_hub<1.0.0,>=0.1.0->datasets==2.0.1.dev0) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\user\\anaconda3\\lib\\site-packages (from huggingface_hub<1.0.0,>=0.1.0->datasets==2.0.1.dev0) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in c:\\users\\user\\anaconda3\\lib\\site-packages (from packaging->datasets==2.0.1.dev0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets==2.0.1.dev0) (2021.10.8)\n",
            "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets==2.0.1.dev0) (3.2)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets==2.0.1.dev0) (1.26.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from requests>=2.19.0->datasets==2.0.1.dev0) (2.0.4)\n",
            "Requirement already satisfied: colorama in c:\\users\\user\\anaconda3\\lib\\site-packages (from tqdm>=4.62.1->datasets==2.0.1.dev0) (0.4.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\user\\anaconda3\\lib\\site-packages (from aiohttp->datasets==2.0.1.dev0) (1.3.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from aiohttp->datasets==2.0.1.dev0) (1.7.2)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in c:\\users\\user\\anaconda3\\lib\\site-packages (from aiohttp->datasets==2.0.1.dev0) (4.0.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\user\\anaconda3\\lib\\site-packages (from aiohttp->datasets==2.0.1.dev0) (1.2.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\user\\anaconda3\\lib\\site-packages (from aiohttp->datasets==2.0.1.dev0) (21.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\user\\anaconda3\\lib\\site-packages (from aiohttp->datasets==2.0.1.dev0) (6.0.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pandas->datasets==2.0.1.dev0) (2021.3)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\user\\anaconda3\\lib\\site-packages (from pandas->datasets==2.0.1.dev0) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in c:\\users\\user\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7.3->pandas->datasets==2.0.1.dev0) (1.16.0)\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "  Running command git clone -q https://github.com/huggingface/datasets.git 'C:\\Users\\User\\AppData\\Local\\Temp\\pip-req-build-oiifhv47'\n"
          ]
        }
      ],
      "source": [
        "! pip install git+https://github.com/huggingface/datasets.git"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FykUK-TFXf-2"
      },
      "source": [
        "For our goals we will use [Datasets](https://huggingface.co/docs/datasets/) library and take `yahoo_answers_topics` dataset - the task of this dataset is to divide documents on 10 topic categories. More detiled information can be found on the dataset [page](https://huggingface.co/datasets/viewer/).\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "ebsFAQsgXNB0"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200,
          "referenced_widgets": [
            "4689ed3126874788bf1ec1aeabc689c4",
            "e8808b6ecc6b49aeb896aa505f50b831",
            "a0c7aa324ee54c65b2e8e099d1ccc212",
            "065cae0e684449ba82815ebd39e48a08",
            "fee97eaaa80b4794bc4ec2f4fbb6524d",
            "242a023e4ee14159885cc486161cd9a6",
            "ea016fe4774449819020c64873ae42ae",
            "064ed687126b4a548c69349444213dd4",
            "002b55b925e840f0ad9cf491dce2b2c3",
            "f2b849756eef4972bbe0309acc93e71a",
            "8fa0f89f3ddd486aa984dce1dd3b8b33",
            "952f0dfbfdbc48aaadd541248370d918",
            "3c63a6d0261f46e3a64b09b6b0120619",
            "5bbc267f0cfb4ea0bf2dc59076653509",
            "823c8a14a0564c43835bfd8690f707a5",
            "c96c1a2ff4d04ace9f9bc1f76547b538",
            "1ee0f62fe7dc426fb9fd0f37cdbec8b1",
            "ac69acbb66b84e7fb249780927ee525c",
            "a5e4167122054541870a76dd2dae4dd6",
            "d15dfb35ee96428690b888d8cfa0e164",
            "ea79830b6b8444c688bf6e8c889e1ded",
            "a9b3b30457c2480dbe6ff826acb6cbce",
            "4d504996b8d84dc8897a763c596eccec",
            "d09b0b11518f497eb053fb3906bc5cf4",
            "46760e479a3642f78015d64dfbacc770",
            "b5eac42a7ee14b0b9f7def444d96f1e6",
            "8d0e619ed0774a41abdb6a9ef5db7a96",
            "75b2f07b56bb482fa3f3212e8efc628f",
            "77b9a4c95c5d4a449bef373fbfb545d2",
            "fac559c605094bf9980ad563c8bc8665",
            "ed2ed3dd67d3437da5837778eb6cb823",
            "5539451cc1624d00b49e11e0f9df713a",
            "915b4f3170dd4c33b7016d08c418fe2e",
            "7f12a6ba25fb4a9b9b96c03ef0fd4bb6",
            "269a8413edaf4d69bc739148394901e4",
            "bdf0a50f3ced4aedb30613ca14a02648",
            "bd658fa357b548819ada046f2206773b",
            "d7d77babe2764ce98750114c12920943",
            "dfd798a5ec7e47ee8b00133ce807c1eb",
            "1d67f22868504b56b2d79dcb0465a239",
            "dad2bef78f98433381db964d767c639d",
            "991e577f5c054278952fb166781d302c",
            "7f89dae5d2564c828fdaba0a2a6fd684",
            "1d415f6f20834c7eb570cbbf458ffa51",
            "46195adf6bd94a249ab04df6878c54b8",
            "f536b30b499246d4932eaeb9abdfcfaf",
            "11680c21182040a39ea564dab32cb243",
            "a44957c1e08f4c9887885593950ce557",
            "9416f30f744249de93276c9ed1fa8709",
            "378c853b3d1445a388af190424b7483b",
            "bcf7663a5f0b48889718932a515f8b22",
            "7af503ff1d884980a71a97dabeb3b86c",
            "dd85342f892849caaa0cfedd96a19372",
            "ec1eb5a0b28046b0bf7bc146382c6f4f",
            "677bf3bd8c564fc280065562aca5a152",
            "c837d4417e7a4342ad1faa6e6d97f45f",
            "71ed4d5d2f604a1484a4b2d9cbe4c663",
            "4dd4b83d9bc24e528c4dbfaff10a7851",
            "0fb38cd3a8fe49fc892b1ebd42c61685",
            "6debe5a643f04fa7808bc3e371f23cf6",
            "e0ee47f5f594486aa30a8b0fd0ac85d7",
            "538f787428684cdbb9f9d94a005de425",
            "7fd3d69be9794181bffd685996f86c62",
            "7cfe75a2932a4eb4b41702250f6a93a6",
            "ee8ac67bba6845dd80f02b72c29b15e9",
            "9319afc8f34a42048f72db5abc0a4511"
          ]
        },
        "id": "UbzxZi42XOUG",
        "outputId": "c9b93941-a5b1-455a-ab9d-84154fd648e8"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading builder script: 3.60kB [00:00, 634kB/s]                    \n",
            "Downloading metadata: 1.88kB [00:00, 316kB/s]                  \n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset yahoo_answers_topics/yahoo_answers_topics (download: 304.68 MiB, generated: 756.38 MiB, post-processed: Unknown size, total: 1.04 GiB) to C:\\Users\\User\\.cache\\huggingface\\datasets\\yahoo_answers_topics\\yahoo_answers_topics\\1.0.0\\0edb353eefe79d9245d7bd7cac5ae6af19530439da520d6dde1c206ee38f4439...\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading data: 100%|██████████| 319M/319M [01:56<00:00, 2.75MB/s]   \n",
            "                                                                                           \r"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataset yahoo_answers_topics downloaded and prepared to C:\\Users\\User\\.cache\\huggingface\\datasets\\yahoo_answers_topics\\yahoo_answers_topics\\1.0.0\\0edb353eefe79d9245d7bd7cac5ae6af19530439da520d6dde1c206ee38f4439. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 2/2 [00:00<00:00, 11.98it/s]\n"
          ]
        }
      ],
      "source": [
        "dataset = load_dataset('yahoo_answers_topics', download_mode=\"force_redownload\") # the result is a dataset dictionary of train and test splits in this case"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6w1fNmzxNP_o",
        "outputId": "7dfd0f7a-fd9b-40c2-b9a8-9d991e27b15b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'topic', 'question_title', 'question_content', 'best_answer'],\n",
              "        num_rows: 1400000\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['id', 'topic', 'question_title', 'question_content', 'best_answer'],\n",
              "        num_rows: 60000\n",
              "    })\n",
              "})"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pNs4hi_-NrSe",
        "outputId": "215510da-a4c5-4918-c683-2cf99d5e84d7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'best_answer': 'Optical mice use an LED and a camera to rapidly capture images of the surface beneath the mouse.  The infomation from the camera is analyzed by a DSP (Digital Signal Processor) and used to detect imperfections in the underlying surface and determine motion. Some materials, such as glass, mirrors or other very shiny, uniform surfaces interfere with the ability of the DSP to accurately analyze the surface beneath the mouse.  \\\\nSince glass is transparent and very uniform, the mouse is unable to pick up enough imperfections in the underlying surface to determine motion.  Mirrored surfaces are also a problem, since they constantly reflect back the same image, causing the DSP not to recognize motion properly. When the system is unable to see surface changes associated with movement, the mouse will not work properly.',\n",
              " 'id': 0,\n",
              " 'question_content': 'or even on some surfaces?',\n",
              " 'question_title': \"why doesn't an optical mouse work on a glass table?\",\n",
              " 'topic': 4}"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset['train'][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fs1NPy2LJNig",
        "outputId": "ec2d6b62-0016-42fa-b13f-a0804677715d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1400000"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "dataset['train']"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4U4YUOB5W8uG"
      },
      "source": [
        "# Fine-tuning the model** (20 points)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ZDYIq9l7CYBR"
      },
      "outputs": [],
      "source": [
        "from transformers import (ElectraTokenizer, ElectraForSequenceClassification,\n",
        "                          get_scheduler, pipeline, ElectraForMaskedLM, ElectraModel,\n",
        "                          Trainer,TrainingArguments, InputFeatures) \n",
        "\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data import Dataset\n",
        "from datasets import load_metric"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "a9nFkpDc3lGu"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ElZ6k36rb0VG"
      },
      "source": [
        "Fine-tuning procedure on the end task consists of adding additional layers on the top of the pre-trained model. The resulting model can be tuned fully (passing gradients through the all model) or partially."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pNEmksaPb3Uu"
      },
      "source": [
        "**Task**: \n",
        "- load tokenizer and model\n",
        "- look at the predictions of the model as-is before any fine-tuning\n",
        "\n",
        "\n",
        "```\n",
        "- Why don't you ask [MASK]?\n",
        "- What is [MASK]\n",
        "- Let's talk about [MASK] physics\n",
        "```\n",
        "\n",
        "- convert `best_answer` to the input tokens (supporting function for dataset is provided below) \n",
        "\n",
        "```\n",
        "def tokenize_function(examples):\n",
        "    return tokenizer(examples[\"best_answer\"], padding=\"max_length\", truncation=True)\n",
        "\n",
        "tokenized_datasets = dataset.map(tokenize_function, batched=True)\n",
        "```\n",
        "\n",
        "- define optimizer, sheduler (optional)\n",
        "- fine-tune the model (write the training loop), plot the loss changes and measure results in terms of weighted F1 score\n",
        "- get the masked word prediction (sample sentences above) on the fine-tuned model, why the results as they are and what should be done in order to change that (write down your answer)\n",
        "- Tune the training hyperparameters (and write down your results).\n",
        "\n",
        "**Tips**:\n",
        "- The easiest way to get predictions is to use transformers `pipeline` function \n",
        "- Do not forget to set `num_labels` parameter, when initializing the model\n",
        "- To convert data to batches use `DataLoader`\n",
        "- Even the `small` version of Electra can be long to train, so you can take data sample (>= 5000 and set seed for reproducibility)\n",
        "- You may want to try freezing (do not update the pretrained model weights) all the layers exept the ones for classification, in that case use:\n",
        "\n",
        "\n",
        "```\n",
        "for param in model.electra.parameters():\n",
        "      param.requires_grad = False\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c2eJ6E3-3viu"
      },
      "source": [
        "### load tokenizer and model\n",
        "Do not forget to set num_labels parameter, when initializing the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "iRLUhVHqUxVC"
      },
      "outputs": [],
      "source": [
        "MODEL_NAME = \"google/electra-small-generator\"\n",
        "TOKENIZER_NAME = \"google/electra-small-generator\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PZd27yS_3zmV",
        "outputId": "e4d528f8-4672-4579-b2fc-a638cb28347b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at google/electra-small-generator were not used when initializing ElectraForSequenceClassification: ['generator_predictions.LayerNorm.bias', 'generator_lm_head.weight', 'generator_predictions.LayerNorm.weight', 'generator_predictions.dense.bias', 'generator_lm_head.bias', 'generator_predictions.dense.weight']\n",
            "- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at google/electra-small-generator and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "tokenizer = ElectraTokenizer.from_pretrained(TOKENIZER_NAME)\n",
        "model = ElectraForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=10)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uj2WWwOX4B3H"
      },
      "source": [
        "### look at the predictions of the model as-is before any fine-tuning\n",
        "The easiest way to get predictions is to use transformers pipeline function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "093875ee0cde4df3bc90a5703bfa32ef",
            "a50d6967b49c465e92a9c85c53dd45eb",
            "63e548f2359f469285dc9a7000f89082",
            "80208d26cd274c768843ba736c911a0c",
            "fc095f16bef443fbb2bac3848b195c9d",
            "2068e394fe364942b73b71dcee3b4a9f",
            "e4b7aff9c8634f04989e0b01362d3705",
            "a5e5eac91d3148f99bbf0572801025fd",
            "db9485e4c39248449317b843c49c134b",
            "8b39cafe63324788afaf2922dab983f0",
            "c9695739961f43adbebf7c88bc32fc6c"
          ]
        },
        "id": "jenKje6S31SG",
        "outputId": "420a0cba-c93e-4924-ffc6-63efeb92f71e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "093875ee0cde4df3bc90a5703bfa32ef",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "predictor = pipeline(\"fill-mask\", model=MODEL_NAME)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NLkvhj-x4cMv",
        "outputId": "a8e02088-d986-491f-c5fe-f7487655d681"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.5342992544174194,\n",
              "  'sequence': \"why don't you ask me?\",\n",
              "  'token': 2033,\n",
              "  'token_str': 'me'},\n",
              " {'score': 0.08196018636226654,\n",
              "  'sequence': \"why don't you ask questions?\",\n",
              "  'token': 3980,\n",
              "  'token_str': 'questions'},\n",
              " {'score': 0.04395333677530289,\n",
              "  'sequence': \"why don't you ask them?\",\n",
              "  'token': 2068,\n",
              "  'token_str': 'them'},\n",
              " {'score': 0.04017288610339165,\n",
              "  'sequence': \"why don't you ask why?\",\n",
              "  'token': 2339,\n",
              "  'token_str': 'why'},\n",
              " {'score': 0.030024440959095955,\n",
              "  'sequence': \"why don't you ask yourself?\",\n",
              "  'token': 4426,\n",
              "  'token_str': 'yourself'}]"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sen_1 = \"Why don't you ask [MASK]?\"\n",
        "predictor(sen_1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3QhrTN4L4o0j",
        "outputId": "bf4561ba-cb18-431a-bb00-a46777243a48"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.4963124096393585,\n",
              "  'sequence': 'what is it?',\n",
              "  'token': 2009,\n",
              "  'token_str': 'it'},\n",
              " {'score': 0.2050585150718689,\n",
              "  'sequence': 'what is this?',\n",
              "  'token': 2023,\n",
              "  'token_str': 'this'},\n",
              " {'score': 0.12763532996177673,\n",
              "  'sequence': 'what is that?',\n",
              "  'token': 2008,\n",
              "  'token_str': 'that'},\n",
              " {'score': 0.015324895270168781,\n",
              "  'sequence': 'what is wrong?',\n",
              "  'token': 3308,\n",
              "  'token_str': 'wrong'},\n",
              " {'score': 0.01502312533557415,\n",
              "  'sequence': 'what is what?',\n",
              "  'token': 2054,\n",
              "  'token_str': 'what'}]"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sen_2 = \"What is [MASK]?\"\n",
        "predictor(sen_2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hm0bVt9H4sUu",
        "outputId": "c19c53bd-198d-4bab-8daa-8ef08090e637"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.24027501046657562,\n",
              "  'sequence': \"let's talk about quantum physics\",\n",
              "  'token': 8559,\n",
              "  'token_str': 'quantum'},\n",
              " {'score': 0.21258601546287537,\n",
              "  'sequence': \"let's talk about theoretical physics\",\n",
              "  'token': 9373,\n",
              "  'token_str': 'theoretical'},\n",
              " {'score': 0.056394025683403015,\n",
              "  'sequence': \"let's talk about particle physics\",\n",
              "  'token': 10811,\n",
              "  'token_str': 'particle'},\n",
              " {'score': 0.0332079641520977,\n",
              "  'sequence': \"let's talk about real physics\",\n",
              "  'token': 2613,\n",
              "  'token_str': 'real'},\n",
              " {'score': 0.022627945989370346,\n",
              "  'sequence': \"let's talk about mathematical physics\",\n",
              "  'token': 8045,\n",
              "  'token_str': 'mathematical'}]"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sen_3 =  \"Let's talk about [MASK] physics\"\n",
        "predictor(sen_3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CBqJJYps5BAf"
      },
      "source": [
        "### Convert best_answer to the input tokens (supporting function for dataset is provided below)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "ipfUJyTp45by"
      },
      "outputs": [],
      "source": [
        "def tokenize_function(examples):\n",
        "    return tokenizer(examples, padding=\"max_length\", truncation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "Lf5RaYdi7GXo"
      },
      "outputs": [],
      "source": [
        "samples_number=5000"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "LyYlv3Rh6-V-"
      },
      "outputs": [],
      "source": [
        "tokenized_train_datasets = tokenize_function(dataset[\"train\"][\"best_answer\"][:samples_number])#.map(tokenize_function, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "UYdQ0qlv6_0n"
      },
      "outputs": [],
      "source": [
        "tokenized_test_datasets = tokenize_function(dataset[\"test\"][\"best_answer\"][:samples_number])#.map(tokenize_function, batched=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "EgaOM3BGHMIa"
      },
      "outputs": [],
      "source": [
        "train_datasets_topics= dataset[\"train\"][\"topic\"][:samples_number]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "yDI_ctMCHT4a"
      },
      "outputs": [],
      "source": [
        "test_datasets_topics= dataset[\"test\"][\"topic\"][:samples_number]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9vBdDsS55Qu2"
      },
      "source": [
        "### define optimizer, sheduler (optional)\n",
        "In order to perform the training, I am using Trainer() and by default it uses optimizer (an instance of AdamW on your model) and a scheduler (given by get_linear_schedule_with_warmup()) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y8hu-qwO5XV3"
      },
      "source": [
        "### fine-tune the model (write the training loop), plot the loss changes and measure results in terms of weighted F1 score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "NXtKB9h5L82J"
      },
      "outputs": [],
      "source": [
        "class dataset_for_trainer (Dataset): #must implement __len__ and __getitem__\n",
        "    def __init__(self, tokenized_data,topics,length):\n",
        "      self.tokenized_data=tokenized_data\n",
        "      self.labels=topics\n",
        "      self.length=length\n",
        "\n",
        "    def __len__(self):\n",
        "      return (self.length)\n",
        "\n",
        "    def __getitem__(self, id):\n",
        "      return InputFeatures(\n",
        "            input_ids=self.tokenized_data['input_ids'][id],\n",
        "            token_type_ids=self.tokenized_data['token_type_ids'][id],\n",
        "            attention_mask=self.tokenized_data['attention_mask'][id],\n",
        "            label=self.labels[id])\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "hCdaEIJLGaCs"
      },
      "outputs": [],
      "source": [
        "training= dataset_for_trainer(tokenized_train_datasets,train_datasets_topics,samples_number)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "PVrKrUONGa4c"
      },
      "outputs": [],
      "source": [
        "evaluating=dataset_for_trainer(tokenized_test_datasets,test_datasets_topics,samples_number)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "gAbBFw0a5WMO"
      },
      "outputs": [],
      "source": [
        "np.random.seed(7)\n",
        "torch.manual_seed(7)\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir=\"/content/drive/My Drive/output\",\n",
        "    overwrite_output_dir=True,\n",
        "    evaluation_strategy=\"steps\",\n",
        "    num_train_epochs=5,\n",
        "    per_device_train_batch_size=8,\n",
        "    do_train=True,\n",
        "    do_eval=True,)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "lq915sKQBv-L"
      },
      "outputs": [],
      "source": [
        "metric = load_metric('f1')\n",
        "\n",
        "def compute_metrics(pred):\n",
        "    predictions, label_ids = pred\n",
        "    preds = np.argmax(predictions, axis=1)\n",
        "    return metric.compute(\n",
        "        predictions=preds, references=label_ids, average=\"weighted\")\n",
        "   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "_Wp0RSbcC2ws"
      },
      "outputs": [],
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=training,\n",
        "    eval_dataset=evaluating,\n",
        "    compute_metrics=compute_metrics\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "8WPPLP_TVX8u",
        "outputId": "0c45a843-8c3a-457b-b842-5e6aa9d21269"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:309: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 5000\n",
            "  Num Epochs = 5\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3125\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3125' max='3125' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3125/3125 26:40, Epoch 5/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.987100</td>\n",
              "      <td>1.467952</td>\n",
              "      <td>0.538092</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.902300</td>\n",
              "      <td>1.543128</td>\n",
              "      <td>0.539431</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.803900</td>\n",
              "      <td>1.619953</td>\n",
              "      <td>0.522479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.659900</td>\n",
              "      <td>1.721062</td>\n",
              "      <td>0.532423</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.556900</td>\n",
              "      <td>1.727321</td>\n",
              "      <td>0.534846</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.453800</td>\n",
              "      <td>1.785547</td>\n",
              "      <td>0.538925</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/My Drive/output/checkpoint-500\n",
            "Configuration saved in /content/drive/My Drive/output/checkpoint-500/config.json\n",
            "Model weights saved in /content/drive/My Drive/output/checkpoint-500/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/My Drive/output/checkpoint-1000\n",
            "Configuration saved in /content/drive/My Drive/output/checkpoint-1000/config.json\n",
            "Model weights saved in /content/drive/My Drive/output/checkpoint-1000/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/My Drive/output/checkpoint-1500\n",
            "Configuration saved in /content/drive/My Drive/output/checkpoint-1500/config.json\n",
            "Model weights saved in /content/drive/My Drive/output/checkpoint-1500/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/My Drive/output/checkpoint-2000\n",
            "Configuration saved in /content/drive/My Drive/output/checkpoint-2000/config.json\n",
            "Model weights saved in /content/drive/My Drive/output/checkpoint-2000/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/My Drive/output/checkpoint-2500\n",
            "Configuration saved in /content/drive/My Drive/output/checkpoint-2500/config.json\n",
            "Model weights saved in /content/drive/My Drive/output/checkpoint-2500/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to /content/drive/My Drive/output/checkpoint-3000\n",
            "Configuration saved in /content/drive/My Drive/output/checkpoint-3000/config.json\n",
            "Model weights saved in /content/drive/My Drive/output/checkpoint-3000/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "train_info = trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u8f5Tj15mq7z",
        "outputId": "41daf134-5c31-49a2-9726-110c1972f3b1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=3125, training_loss=0.7166390466308594, metrics={'train_runtime': 1600.5695, 'train_samples_per_second': 15.619, 'train_steps_per_second': 1.952, 'total_flos': 735648921600000.0, 'train_loss': 0.7166390466308594, 'epoch': 5.0})"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_info"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "id": "ONu3AHYUo4vl",
        "outputId": "6e801f8d-9f1f-4063-c480-008272f56cae"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'Training Loss')"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3xUVd7H8c8vCQkt9IA06UUQEQhIESIiRRQVsYAVRVEBEUF3cVcffdx1191HEEGKqCCyCqKiYkVEAWlK6E0gFOkQOkiH8/wxFzfLJjCETG4m832/Xvc1M/femfmeDOSXe86Ze805h4iIRK4ovwOIiIi/VAhERCKcCoGISIRTIRARiXAqBCIiES7G7wAXqkSJEq5ixYp+xxARCSvz58/f5ZxLSG9b2BWCihUrkpyc7HcMEZGwYma/ZrQtZF1DZjbKzHaa2bIMtpuZDTazFDNbYmb1Q5VFREQyFsoxgneAdufYfj1QzVu6A8NDmEVERDIQskLgnJsB7DnHLjcD77qAuUARMysdqjwiIpI+P2cNlQU2pXm82Vv3X8ysu5klm1lyampqtoQTEYkUYTF91Dk30jmX6JxLTEhId9BbREQyyc9CsAUon+ZxOW+diIhkIz8LwSTgPm/2UGNgv3Num495REQiUsi+R2Bm44BrgBJmthl4HsgD4JwbAXwFtAdSgMPAA6HKAvDL9gN8s2w73a6uRHzePKF8KxGRsBKyQuCc63Ke7Q7oGar3P9u0VakM+m4N78zewKNJVbivSQXyx4bd9+lERLJcWAwWZ4VHk6owqVczrixfhJe//oUW/5zG6FnrOXrilN/RRER8ZeF2hbLExER3saeYSN6wh1e+XcXcdXsoXTgvj19bjdsTy5EnOmLqoohEGDOb75xLTHdbJBYCAOccs9fu5pVvV7Fw4z4uLZafPtdV4+YryxIdZVmQVEQk5zhXIYjYP4HNjGZVSzDxsaaM6ppIwbgY+k5YTNtBM/hyyTZOnw6vAikiklkRWwjOMDOurVmKLx6/muF3B8571/P9Bdw4ZCZTV+4g3I6YREQuVMQXgjOioozr65Rmcp8WvHpnXX47fpJuY5LpOGw2M9fsUkEQkVwrYscIzufEqdN8PH8zg6euYev+o1xVqRhPta1Bw4rFQv7eIiJZTYPFF+HYyVOM/3kTr/+QQurBYyRVT6Bfm+pcUa5ItmUQEblYKgRZ4MjxU7w7ZwMjpq9l7+ETtKlVir5tqlPzkkLZnkVE5EKpEGShg0dPMHrWBt6csY5Dx09y4xVl6HNdNaokFPQtk4jI+agQhMC+w8cZOWMdo2dt4NjJU3SqX47erapRvlh+v6OJiPwXFYIQ2nXoGMOnrWXs3F9xznFnw/L0almNSwrn9TuaiMjvVAiywfb9R3n9hzWM/3kT0VHGvY0r8Og1VShRMM7vaCIiKgTZadOew7w2dQ0TF2wmb55oHmhWke7Nq1A4v059LSL+USHwwdrUQwz6bg2fL95KfN4YHm5emQeaVdS1EETEFyoEPlq57QADp6xmyoodFM2fx7sWQkXyxUb7HU1EIogKQQ6weNM+BkxZzYzVqSTEx9GrZVU6NypPXIwKgoiEngpBDvLz+sC1EH5ev4cyhfPSu1U1OjXQtRBEJLR0GuocpFGlYnzQvTH/6nYVCYXy0n/iUloPnM6nC7dwSqe+FhEfqBD4wMy4uloJPu3RlLfuSyRfbAx9PlhEu0Ez+HqproUgItlLhcBHZsZ1tUrx5eNXM/Su+px2jsfeW0CH12fy/S+6FoKIZA8VghwgKsq44YrSfPtkEgNur8uBoyd48J1kOg2fzeyUXX7HE5FcToPFOdCJU6f5MHkzQ75fw7b9R2lSuThPta1Ogwq6FoKIZI5mDYWpoydOMe7njQz9IYVdh45zTY0E+rWuQZ1yhf2OJiJhRoUgzB0+fpIxs39lxPS17D9ygna1L+HJ1tWpcUm839FEJEyoEOQSB46eYNTM9bz143p+O36Sm+qWoc911alUooDf0UQkh1MhyGX2/nackT+u451ZGzh+6jSd6peld6tqlCuqayGISPpUCHKp1IPHGDYthffmbsTh6NLoUnq2rEqpQroWgoj8J9++WWxm7cxslZmlmFn/dLZXMLOpZrbEzKaZWblQ5sltEuLjeL5DbaY9fQ23J5bn/Z820uKfP/DSlyvYfeiY3/FEJEyE7IjAzKKB1UBrYDMwD+jinFuRZp8PgS+cc2PM7FrgAefcved6XR0RZGzj7sMMmrqaTxduIW+eaB5sVomHm1fWtRBExLcjgkZAinNunXPuODAeuPmsfWoB33v3f0hnu1yAS4vnZ+AdV/Ltky1oWbMkr/+QQtIrP/DeT7/qPEYikqFQFoKywKY0jzd769JaDNzq3e8IxJtZ8RBmighVS8Yz9K76fNn7amqUiufPnyyj47BZLNq0z+9oIpID+X2KiaeAJDNbCCQBW4BTZ+9kZt3NLNnMklNTU7M7Y9iqXaYw47s35rXOV7J9/1E6DptF/4+XsOe3435HE5EcJJRjBE2AF5xzbb3HzwA45/6ewf4FgV+cc+ccMNYYQeYcPHqC175bw+jZGygYF8PTbWvQpdGlREeZ39FEJBv4NUYwD6hmZpXMLBboDEw6K1gJMzuT4RlgVAjzRLT4vHl49sZafP1Ecy4rHc+zny7jlqGzWLhxr9/RRMRnISsEzrmTQC9gMrASmOCcW25mL5rZTd5u1wCrzGw1UAp4KVR5JKB6qXjGPdyYwV3qsePAUToOm80fP1qi6aYiEUxfKItgh46dZPDUNYyauZ4CcTE81bYGd6m7SCRX0qUqJV0F42L4U/vL+PqJ5tQqXYjnPl3GzUNnskDdRSIRRYVAqFYqnvcfvorBXeqRevAYtw6bzR8+WqzuIpEIoUIgQOCymTfVLcPUftfwSIvKTFywhZavTGPsnA36MppILqdCIP+hYFwMz3jdRZeXLcxzny3nptdnMv9XdReJ5FYqBJKuaqXiee+hqxjSpR67Dh2j0/DZPP3hYnapu0gk11EhkAyZGR3SdBd9snAL174yjXfnqLtIJDdRIZDzOtNd9E2f5tQpV5j/+Ww5HYbMZP6ve/yOJiJZQIVAgla1ZDz/6nYVQ++qz57fjtNp+ByeUneRSNhTIZALYmbccEVppvZL4tGkKny6MDC76J1Z6zl56rTf8UQkE1QIJFMKxMXQ//qafNOnBXXLFeGFz1fQ4fVZJG9Qd5FIuFEhkItStWRBxnZrxLC767Pv8HFuGzGHfhMWk3pQ3UUi4UKFQC6amdG+Tmm+65vEY9dUYdLiLVw7QN1FIuFChUCyTIG4GP7YLtBddGX5QHfRjUNmMk/dRSI5mgqBZLkqCQV598FAd9GBIye4fcQc+k5YpO4ikRxKhUBC4vfuon5J9LimCp8v3sq1r0xjtLqLRHIcFQIJqfyxMfzhTHfRpUX4X6+76Of16i4SySlUCCRbnOkuGnFPoLvojjfm8OQHi9h58Kjf0UQingqBZBszo93lge6ini2r8OWSbbR6ZTpvz1R3kYifVAgk2+WPjeHptjX5pk9z6lUoyl++CHQX/bRut9/RRCKSCoH4pnJCQcY80JAR9zTg4NGT3DlyLn3GL2TnAXUXiWQnFQLxVaC76BK+65tEr5ZV+Wrpdq4dMJ23flzHCXUXiWQLFQLJEfLFRvNU2xpMfrIFDSoU5a9fruTGweouEskOKgSSo1QqUYB3HmjIG/c24NCxQHfRE+MXskPdRSIho0IgOY6Z0bZ2oLvo8Wur8vXS7Vz7yjR1F4mEiAqB5Fj5YqPp1ybQXdSwUjH++uVKbhj8I3PWqrtIJCupEEiOV6lEAUZ3bcjIexvw27FTdHlzLr3HqbtIJKuoEEhYMDPaeN1Fva+tyjfLA91Fb85Qd5HIxVIhkLCSLzaavm1q8G2fFjSqVIyXvlpJ+9d+ZPbaXX5HEwlbIS0EZtbOzFaZWYqZ9U9n+6Vm9oOZLTSzJWbWPpR5JPeoWKIAo7o25M37Ejly4hR3vfkTj49byPb96i4SuVAhKwRmFg0MBa4HagFdzKzWWbs9C0xwztUDOgPDQpVHch8zo3WtUoHuolbVmLx8O60GTGPsnA1+RxMJKxdUCMwsyswKBbl7IyDFObfOOXccGA/cfNY+DjjzeoWBrReSRwQgb55o+rauzpQnW9CgYjGe+2w5//v5ck6ddn5HEwkL5y0EZva+mRUyswLAMmCFmT0dxGuXBTalebzZW5fWC8A9ZrYZ+Ap4PKjUIumoULwA73RtSLerKzF61gZ6vDefoydO+R1LJMcL5oiglnPuAHAL8DVQCbg3i96/C/COc64c0B4Ya2b/lcnMuptZspklp6amZtFbS24UFWU8d2Mt/ufGWny7Ygdd3pzL7kO6RKbIuQRTCPKYWR4ChWCSc+4EgS6d89kClE/zuJy3Lq1uwAQA59wcIC9Q4uwXcs6NdM4lOucSExISgnhriXQPXl2J4XfXZ8XWA3QaPpsNu37zO5JIjhVMIXgD2AAUAGaYWQXgQBDPmwdUM7NKZhZLYDB40ln7bARaAZjZZQQKgf7klyzR7vLSvP9wY/YfOcGtw2ezYONevyOJ5EjnLQTOucHOubLOufYu4FegZRDPOwn0AiYDKwnMDlpuZi+a2U3ebv2Ah81sMTAO6Oqc0wifZJkGFYoysUcz4vPG0GXkXL5Ztt3vSCI5jp3v966ZPQGMBg4CbwH1gP7OuW9DH++/JSYmuuTkZD/eWsLY7kPH6DYmmcWb9/H8jbXo2qyS35FEspWZzXfOJaa3LZiuoQe9weI2QFECA8UvZ2E+kZArXjCOcQ83pvVlpXjh8xX89YsVnNb0UhEguEJg3m17YKxzbnmadSJhI19sNMPvaUDXphV5a+Z6eo1boOmlIgRXCOab2bcECsFkM4sHdJYvCUvRUcbzHWrx7A2X8dXS7dzz1k/s/e2437FEfBVMIegG9AcaOucOA7HAAyFNJRJCZsZDzSsz9K76LNmyn07DZ7Nx92G/Y4n4JphZQ6cJfAfgWTN7BWjqnFsS8mQiIXbDFaV576Gr2HP4OB2HzWLRpn1+RxLxRTCnmHgZeAJY4S29zexvoQ4mkh0aVizGx481JX9cNJ1HzmHKih1+RxLJdsF0DbUHWjvnRjnnRgHtgBtDG0sk+1RJKMjEx5pRo1Q8j4xN1tlLJeIEe/bRImnuFw5FEBE/JcTHMa57Y66tWZLnPlvO379eqemlEjGCKQR/Bxaa2TtmNgaYD7wU2lgi2S9/bAxv3JvIvY0r8Mb0dfQev1DTSyUixJxvB+fcODObBjT0Vv0RqBDKUCJ+iY4yXry5NuWK5uPvX//CzoPHGHlvA4rkj/U7mkjIBNU15Jzb5pyb5C3bgQ9DnEvEN2bGI0lVGNylHos27qPT8Nls2qPppZJ7ZfZSlfpmseR6N9Utw9hujUg9eIyOw2azdPN+vyOJhERmC4FG0SQiXFW5OBN7NCUuJoo73pjD979oeqnkPhmOEZjZ56T/C9+A4iFLJJLDVC0Zzyc9m/LgO/N4aEwyf72lDndddanfsUSyzLkGi1/J5DaRXKdkfF4+6N6EXu8v4E+fLGXz3sM81aYGUVHqJZXwl2EhcM5Nz84gIjldgbgY3rwvkec+W86waWvZuu8I/7ytLrExme1hFckZzjt9VET+LSY6ir91vJxyRfPxf5NXsePAMUbc24DC+fL4HU0k0/SnjMgFMjN6tqzKoDuvJPnXPdw+YjZb9h3xO5ZIpqkQiGTSLfXKMubBRmzbf5SOQ2exfKuml0p4Cubso5+b2aSzlrFm9oSZ5c2OkCI5VdMqJfj4sabERBl3jJjD9NWpfkcSuWDBHBGsAw4Bb3rLAQIXsq/uPRaJaNVLxfNJz2ZUKF6AB9+ZxwfzNvodSeSCBDNY3NQ51zDN48/NbJ5zrqGZLQ9VMJFwUqpQXiY82oQe7y3gjx8vZcveIzzZujpmml4qOV8wRwQFzez3b8949wt6D3WxVxFPwbgY3r4/kTsTyzP4+xT6fbiY4yd1eW/J+YI5IugHzDSztQS+VVwJ6GFmBYAxoQwnEm7yREfxcqc6lCuajwFTVrPzwDGG3VOfQnk1vVRyLnPu/KcNMrM4oKb3cJVz7mhIU51DYmKiS05O9uvtRYL28fzN/PHjJVQtWZDRDzSkdOF8fkeSCGZm851zieltC3b6aAOgNlAXuMPM7suqcCK5VacG5RjzYCO27D1Cx6GzWbntgN+RRNIVzPTRsQTOLXQ1gYvTNATSrSoi8p+aVS3Bh481AeD2EXP4cY2ml0rOc96uITNbCdRywfQhZQN1DUk42rb/CA+MnkfKzkP8/dY63J5Y3u9IEmEutmtoGXBJJt+4nZmtMrMUM+ufzvZXzWyRt6w2s32ZeR+RnK504Xx8+GgTmlQpztMfLeG179aQQ/62Eglq1lAJYIWZ/QwcO7PSOXfTuZ5kZtHAUKA1sBmYZ2aTnHMr0rzGk2n2fxyod2HxRcJHfN48jOrakGcmLuXV71azZd9hXupYhzzROtOL+CuYQvBCJl+7EZDinFsHYGbjgZuBFRns3wV4PpPvJRIW8kRH8X+3XUHZIvl4beoatu0/yrC76xOv6aXio/MWgou4LkFZYFOax5uBq9Lb0cwqEPh+wveZfC+RsGFmPNm6OmWL5uNPE5dyxxtzeeeBhpQqpFN3iT8yPCY1s5ne7UEzO5BmOWhmWT0PrjPwkXPuVAZZuptZspklp6Zq1oXkDncklmdU14Zs3P0bHYfOYtX2g35HkgiVYSFwzl3t3cY75wqlWeKdc4WCeO0tQNqpEeW8denpDIw7R5aRzrlE51xiQkJCEG8tEh5aVE9gwqNNOOUct42YzeyUXX5HkggU1CiVmUWbWRkzu/TMEsTT5gHVzKySmcUS+GU/KZ3XrgkUBeZcSHCR3KJ2mcJ80qMZZQrn4/7RP/PJws1+R5IIE8wXyh4HdgBTgC+95YvzPc85dxLoBUwGVgITnHPLzexFM0s746gzMD6nfE9BxA9liuRjwqNNaFixGE9+sJjXv9f0Usk+wXyhLAW4yjm3O3sinZu+UCa52fGTp+n/8RImLtxCl0bl+cvNlxOj6aWSBc71hbJgpo9uAnQNPpFsEBsTxYA76lK2aD6GfJ/Ctv1HGXpXfQrEBfNfVSRzgvnXtQ6YZmZf8p9fKBsYslQiEczM6NemBmWK5OPZT5dx58g5jLq/ISU1vVRCJJhjzo0Exgdigfg0i4iEUJdGl/LW/YmsS/2NjsNms2aHppdKaAR1PYKcRGMEEmmWbdnPA+/M49iJU4y8L5HGlYv7HUnCUKZOOmdmg7zbz81s0tlLqMKKyH+6vGxhJj7WlJKF8nLf2z8zafFWvyNJLnOuMYKx3u0r2RFERDJWvlh+Pn60Kd3HJtN73EK27jvCIy0qY2Z+R5NcIMNC4Jyb791m9lxDIpKFCufPw7vdGvHUh0t4+etf2Lz3MC90qK3ppXLRzjtryMyqAX8HagG/T1twzlUOYS4RSUdcTDSv3XklZYvkY8T0tWzff5TBXeqRP1bTSyXzgvlTYjQwHDgJtATeBf4VylAikrGoKKP/9TX5yy2X8/0vO+k8ci6pB4+d/4kiGQimEORzzk0lMMPoV+fcC8ANoY0lIudzb+MKvHlfImt2HKLtoBlMXLBZp6WQTAmmEBwzsyhgjZn1MrOOQMEQ5xKRILS6rBSf9mxGxeL56TthMfe8/RPrd/3mdywJM8EUgieA/EBvoAFwD3B/KEOJSPBqXBLPR4825a+3XM6SzftpO2gGQ6au4fjJ035HkzBxzkLgXXf4TufcIefcZufcA865Ts65udmUT0SCEBVl3NO4AlP7JtG6VikGTFlN+8E/Mm/DHr+jSRg41xfKYrwrhl2djXlE5CKULJSXoXfVZ3TXhhw5forbR8zhmYlL2H/4hN/RJAc71xHBz97tQu/bxPea2a1nluwIJyKZ07JmSab0bUH3FpWZkLyZVgOn8dmiLRpMlnQFM0aQF9gNXAvcCHTwbkUkB8sfG8Of2l/GpF7NKFskH0+MX8T9o+excfdhv6NJDpPhSefMbDMwEDDAebdnOL9OQ62TzolcuFOnHWPnbOCVb1dz8vRpnmhVnYeaVyKPvpUcMTJ10jkgmsA00YIETjtd8KxFRMJEdJTRtVklpvRtQVL1BP7xzS90GDKTBRv3+h1NcoBzHREscM7Vz+Y856UjApGL9+3y7Tw/aTnbDxzl7qsu5Q/talIobx6/Y0kIZfaIQKc1FMml2tS+hCl9k+jatCLv/7SR6wZM56ul2zSYHKHOVQhaZVsKEcl2BeNieL5DbT7t2YySheLo8d4Cuo1JZvNeDSZHmgwLgXNO30QRiQBXlCvCpz2a8ewNlzF33W5aD5zBmzPWcfKUvpkcKTRlQESIiY7ioeaVmdI3iWZVi/PSVyu56fVZLN60z+9okg1UCETkd2WL5OPN+xIZfnd9dh06Rsdhs3hh0nIOHtU3k3MzFQIR+Q9mxvV1SvNdvyTuaVyBMXM20HrgDCYv3+53NAkRFQIRSVehvHl48ebLmfhYU4rkz8MjY+fz8LvJbN13xO9oksVUCETknOpdWpTPH7+aZ66vyY9rUmk9cDqjZq7n1GlNNc0tVAhE5LzyREfxSFIVpjyZRGLFYrz4xQo6DpvFsi37/Y4mWUCFQESCVr5Yft55oCFDutRj676j3PT6TP76xQp+O3bS72hyEUJaCMysnZmtMrMUM+ufwT53mNkKM1tuZu+HMo+IXDwzo0PdMkztm0TnRpfy1sz1tHl1BlNX7vA7mmRSyAqBd3WzocD1QC2gi5nVOmufasAzQDPnXG2gT6jyiEjWKpw/D3/rWIePHm1Cgbhouo1Jpsd789lx4Kjf0eQChfKIoBGQ4pxb55w7DowHbj5rn4eBoc65vQDOuZ0hzCMiIZBYsRhfPN6cp9vWYOrKnVw3YDpj52zQYHIYCWUhKAtsSvN4s7curepAdTObZWZzzaxdei9kZt3NLNnMklNTU0MUV0QyKzYmip4tqzK5Twvqli/Cc58tp9Pw2azcdsDvaBIEvweLY4BqwDVAF+BNMyty9k7OuZHOuUTnXGJCQkI2RxSRYFUsUYCx3Rox6M4r2bTnMB2GzOTlr3/hyPFTfkeTcwhlIdgClE/zuJy3Lq3NwCTn3Ann3HpgNYHCICJhysy4pV5ZvuubRKf65RgxfS2tX53OtFXq+c2pQlkI5gHVzKySmcUCnYFJZ+3zKYGjAcysBIGuonUhzCQi2aRogVj+cdsVfNC9MXExUXQdPY/Hxy1k50ENJuc0ISsEzrmTQC9gMrASmOCcW25mL5rZTd5uk4HdZrYC+AF42jm3O1SZRCT7XVW5OF890Zwnr6vO5GXbuW7AdN7/aSOnNZicY2R4qcqcSpeqFAlfa1MP8edPljJ33R4SKxTlb7fWoXqpeL9jRYTMXqpSRCRLVUkoyLiHG/N/t11BSuohbhj8I69MXsXRExpM9pMKgYhkKzPj9sTyTO2bxE11y/L6Dym0GzSDmWt2+R0tYqkQiIgviheMY8AddXn/oaswM+55+yee/GARuw8d8ztaxFEhEBFfNa1agq+faE7va6vyxZKttBo4nQnJmwi38ctwpkIgIr7Lmyeavm1q8FXv5lQrWZA/fLSEziPnkrLzkN/RIoIKgYjkGNVKxfNB9ya8fGsdVm47QPvXfuTVKas1mBxiKgQikqNERRmdG13K1H7XcH2dS3ht6hrav/Yjc9bqK0ahokIgIjlSQnwcr3Wux7sPNuLkaUeXN+fy9IeL2fvbcb+j5ToqBCKSo7WonsDkPi147JoqfLJwC60GTmfigs0aTM5CKgQikuPli43mj+1q8kXvq6lYPD99Jyzmnrd/Yv2u3/yOliuoEIhI2Kh5SSE+erQpf7nlcpZs2k/bQTN4/fs1HD952u9oYU2FQETCSlSUcW/jCkztl0Try0rxyreruWHwj8zbsMfvaGFLhUBEwlLJQnkZend9RnVN5PDxU9w+Yg7PTFzK/sMn/I4WdlQIRCSsXVuzFFP6tuDh5pWYkLyJVgOnM2nxVg0mXwAVAhEJe/ljY/jzDbX4rGczyhTJS+9xC+k6eh6b9hz2O1pYUCEQkVzj8rKF+aRHM57vUIvkDXto/ep0Rkxfy4lTGkw+FxUCEclVoqOMB5pVYkrfJJpXS+Dlr3+hw5CZLNy41+9oOZYKgYjkSmWK5OPN+xJ5494G7Dt8gluHz+Z/PlvGwaMaTD6bCoGI5Gpta1/ClL4tuL9JRcbO/ZXrBk7nm2XbNJichgqBiOR68Xnz8MJNtfm0RzOKFYjj0X8t4OF357N13xG/o+UIKgQiEjHqli/C572a8af2NZmVsovrBk7n7ZnrOXU6so8OVAhEJKLEREfRvUUVvn2yBY0qFeMvX6zglqGzWLZlv9/RfKNCICIRqXyx/Izu2pDX76rH9gNHuen1mfzlixX8duyk39GynQqBiEQsM+PGK8rwXd8kujS6lLdnrqfNqzOYunKH39GylQqBiES8wvny8FLHOnz8WBMKxEXTbUwyPd6bz44DR/2Oli1UCEREPA0qFOOLx5vzdNsaTF25k+sGTGfsnA25fjBZhUBEJI3YmCh6tqzK5D4tqFu+CM99tpxOw2ezctsBv6OFjAqBiEg6KpYowNhujRh055Vs2nOYDkNm8vLXv3Dk+Cm/o2W5kBYCM2tnZqvMLMXM+qezvauZpZrZIm95KJR5REQuhJlxS72yfNc3iVvrl2XE9LW0GTSd6atT/Y6WpUJWCMwsGhgKXA/UArqYWa10dv3AOXelt7wVqjwiIplVtEAs/7ytLuO7NyZPdBT3j/qZ3uMWknrwmN/RskQojwgaASnOuXXOuePAeODmEL6fiEhINa5cnK+faE6f66rxzbLttBowjXE/b+R0mA8mh7IQlAU2pXm82Vt3tk5mtsTMPjKz8um9kJl1N7NkM0tOTc1dh2QiEl7iYqLpc111vnqiOZeVLsQzE5dy58g5rNlx0O9omeb3YPHnQEXn3BXAFGBMejs550Y65xKdc4kJCQnZGlBEJD1VSxZkfPfG/PO2K1iz8xDtB//IgG9XcerxFRwAAAjMSURBVPRE+A0mh7IQbAHS/oVfzlv3O+fcbufcmU62t4AGIcwjIpKlzIw7EssztW8SN15RhiHfp3D9az8yO2WX39EuSCgLwTygmplVMrNYoDMwKe0OZlY6zcObgJUhzCMiEhLFC8bx6p1XMrZbI047x11v/US/CYvZ89txv6MFJWSFwDl3EugFTCbwC36Cc265mb1oZjd5u/U2s+VmthjoDXQNVR4RkVBrXi2ByX1a0LNlFT5btIVWA6bx0fzNOf4iOJbTA54tMTHRJScn+x1DROScVu84yDMTlzL/1700rVKclzrWoVKJAr7lMbP5zrnE9Lb5PVgsIpIrVS8Vz4ePNOGljpezdMt+2g6awZCpazh+8rTf0f6LCoGISIhERRl3X1WBqX2TaF2rFAOmrKb94B+Zt2GP39H+gwqBiEiIlSyUl6F31Wd014YcOX6K20fMof/HS9h/+ITf0QAVAhGRbNOyZkmm9G3BIy0q8+H8zbQaOI3PFm3xfTBZhUBEJBvlj43hmfaXMalXM8oWyccT4xdx/+h5bNx92LdMKgQiIj6oXaYwE3s044UOtZi/YQ9tBk1n+LS1nDiV/YPJKgQiIj6JjjK6NqvEd/2SSKqewD+++YUOQ2ayYOPebM2hQiAi4rPShfPxxr2JjLy3AfuPnKDT8Nk89+kyDhzNnsFkFQIRkRyiTe1LmNI3ia5NK/LeT79y3YDpfLV0W8gHk1UIRERykIJxMTzfoTaf9mxGQnwcPd5bwENjktmy70jI3lOFQEQkB7qiXBE+69mMZ2+4jNlrd9N64HQ+X7w1JO+lQiAikkPFREfxUPPKTOnbgqZVSoTsXEUxIXlVERHJMuWK5uet+9M9X1yW0BGBiEiEUyEQEYlwKgQiIhFOhUBEJMKpEIiIRDgVAhGRCKdCICIS4VQIREQinPl9ZZwLZWapwK+ZfHoJYFcWxgkHanNkUJsjw8W0uYJzLiG9DWFXCC6GmSU750L39bwcSG2ODGpzZAhVm9U1JCIS4VQIREQiXKQVgpF+B/CB2hwZ1ObIEJI2R9QYgYiI/LdIOyIQEZGzqBCIiES4XFcIzGyDmS01s0VmluytK2ZmU8xsjXdb1FtvZjbYzFLMbImZ1fc3fXDMbJSZ7TSzZWnWXXAbzex+b/81Zna/H20JVgZtfsHMtnif9SIza59m2zNem1eZWds069t561LMrH92tyNYZlbezH4wsxVmttzMnvDW59rP+Rxtzs2fc14z+9nMFntt/l9vfSUz+8nL/4GZxXrr47zHKd72imleK92fRVCcc7lqATYAJc5a90+gv3e/P/AP73574GvAgMbAT37nD7KNLYD6wLLMthEoBqzzbot694v63bYLbPMLwFPp7FsLWAzEAZWAtUC0t6wFKgOx3j61/G5bBu0tDdT37scDq7125drP+Rxtzs2fswEFvft5gJ+8z28C0NlbPwJ4zLvfAxjh3e8MfHCun0WwOXLdEUEGbgbGePfHALekWf+uC5gLFDGz0n4EvBDOuRnAnrNWX2gb2wJTnHN7nHN7gSlAu9Cnz5wM2pyRm4Hxzrljzrn1QArQyFtSnHPrnHPHgfHevjmOc26bc26Bd/8gsBIoSy7+nM/R5ozkhs/ZOecOeQ/zeIsDrgU+8taf/Tmf+fw/AlqZmZHxzyIoubEQOOBbM5tvZt29daWcc9u8+9uBUt79ssCmNM/dzLn/4eVkF9rG3NL2Xl5XyKgz3STksjZ7h//1CPy1GBGf81lthlz8OZtZtJktAnYSKNRrgX3OuZPeLmnz/942b/t+oDgX2ebcWAiuds7VB64HeppZi7QbXeA4KlfPmY2ENnqGA1WAK4FtwAB/42Q9MysIfAz0cc4dSLstt37O6bQ5V3/OzrlTzrkrgXIE/oqvmd0Zcl0hcM5t8W53Ap8Q+MHuONPl493u9HbfApRP8/Ry3rpwdKFtDPu2O+d2eP+JTgNv8u9D4VzRZjPLQ+AX4nvOuYne6lz9OafX5tz+OZ/hnNsH/AA0IdC1F+NtSpv/97Z52wsDu7nINueqQmBmBcws/sx9oA2wDJgEnJktcT/wmXd/EnCfN+OiMbA/zWF3uLnQNk4G2phZUe9Qu423LmycNZ7TkcBnDYE2d/ZmWFQCqgE/A/OAat6MjFgCg22TsjNzsLx+37eBlc65gWk25drPOaM25/LPOcHMinj38wGtCYyN/ADc5u129ud85vO/DfjeOzLM6GcRHL9HzbNyITBLYLG3LAf+7K0vDkwF1gDfAcXcv0fshxLok1sKJPrdhiDbOY7AIfIJAn2B3TLTRuBBAoNKKcADfrcrE20e67VpifcfoXSa/f/stXkVcH2a9e0JzEZZe+bfR05cgKsJdPssARZ5S/vc/Dmfo825+XO+AljotW0Z8D/e+soEfpGnAB8Ccd76vN7jFG975fP9LIJZdIoJEZEIl6u6hkRE5MKpEIiIRDgVAhGRCKdCICIS4VQIREQinAqBSBDM7M/e2SGXeGfAvMrM+phZfr+ziVwsTR8VOQ8zawIMBK5xzh0zsxIEzmo5m8B8/V2+BhS5SDoiEDm/0sAu59wxAO8X/21AGeAHM/sBwMzamNkcM1tgZh9658w5c42Mf1rgOhk/m1lVvxoikh4VApHz+xYob2arzWyYmSU55wYDW4GWzrmW3lHCs8B1LnDSw2Sgb5rX2O+cqwO8DgzK7gaInEvM+XcRiWzOuUNm1gBoDrQEPkjnqleNCVwcZFbglDnEAnPSbB+X5vbV0CYWuTAqBCJBcM6dAqYB08xsKf8+8dcZRuACMF0yeokM7ov4Tl1DIudhZjXMrFqaVVcCvwIHCVxSEWAu0OxM/793JtzqaZ5zZ5rbtEcKIr7TEYHI+RUEhninCz5J4MyP3YEuwDdmttUbJ+gKjDOzOO95zxI4AyZAUTNbAhzznieSY2j6qEiImdkGNM1UcjB1DYmIRDgdEYiIRDgdEYiIRDgVAhGRCKdCICIS4VQIREQinAqBiEiE+38W7nYOdiFLzwAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "steps=[500,1000 ,1500 ,2000 ,2500 ,3000 ]\n",
        "loss= [0.987100,0.902300 , 0.803900,0.659900 , 0.556900, 0.453800 ]\n",
        "plt.plot(steps, loss)\n",
        "plt.xlabel(\"Step\")\n",
        "plt.ylabel(\"Training Loss\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "MY7ZuJGBWCAm",
        "outputId": "e40e2a05-e3d1-42ec-9542-b7c7e35e336b"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'Validation Loss')"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEJCAYAAACOr7BbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3wVVfrH8c9DEjqE3vtKFaVFirqKujbEuv5WxIJYcFFX3fXnb911V9R1Xbe56qIiIqI0uyvYsYENMKFJ710hSAs95fn9cQeN7E24hNxMcvN9v173lbln5s59TgbmyZyZc465OyIiIoeqEHYAIiJSOilBiIhIVEoQIiISlRKEiIhEpQQhIiJRKUGIiEhUcU0QZjbazDab2fwC1qea2WQzm2tmC8xscL51g8xsWfAaFM84RUTkv1k8+0GY2SnALuB5d+8cZf3vgVR3/62Z1QeWAI2A6kA6kAY4kAH0cPdtcQtWRER+JDmeO3f3aWbWqrBNgBpmZkSSwlYgBzgbmOLuWwHMbApwDjCxsO+rV6+et2pV2NeJiEh+GRkZW9y9frR1cU0QMRgOTAI2AjWAy9w9z8yaAuvybbceaBptB2Y2BBgC0KJFC9LT0+MbsYhIAjGzNQWtC/sm9dnAHKAJ0BUYbmY1j2QH7j7S3dPcPa1+/ahJUEREiiDsBDEYeM0jlgOrgA7ABqB5vu2aBWUiIlJCwk4Qa4EzAMysIdAeWAm8B5xlZrXNrDZwVlAmIiIlJK73IMxsItAXqGdm64FhQAqAu48A/gSMMbOvAQN+6+5bgs/+Cfgq2NX9B29Yi4hIyYj3U0yXH2b9RiJXB9HWjQZGxyMuERE5vLCbmEREpJRSghARkaiUIEREyrDpK79jxNQVcdl32B3lRESkCDbt3Mef31rEpLkbaVGnKoP6tKJKxaRi/Q4lCBGRMuRATh7Pfr6Kxz5cRnaec+sZbRl66k+KPTmAEoSISJnx2bItDJs0nxWZu/lZxwbc0/9YWtStGrfvU4IQESnlNmzfywNvLuSd+d/Ssm5VRl+TxukdGsb9e5UgRERKqf05uTw9bSXDP14OwP+e1Y7rf9qGyinF35wUjRKEiEgp9PHizdw3eQGrv9vDuZ0b8Yf+nWhaq0qJxqAEISJSiqz9bg/3v7mADxZtpk39aoy9ric/bRvOSNVKECIipcC+7Fye+GQFI6auILmC8btzOzD4pNZUTA6vu5oShIhIiNyd9xdu4k9vLmT9tr1c0KUJv+/XkUaplcMOTQlCRCQsKzN3cd/khUxdmkn7hjV4YUhverepG3ZY31OCEBEpYXsO5PDvj5Yz6tOVVE5O4p7+nbiqT0tSkkrX6EdKECIiJcTdeevrb/jzW4v4Zsc+ft69GXed24H6NSqFHVpUShAiIiVg2aYshk1awBcrvuPYJjUZPrAbPVrWCTusQilBiIjEUda+bB79YBljvlhNtUrJ/Omizgzs2YKkChZ2aIelBCEiEgfuzn/mbODBtxezZdd+BpzQnDvP7kCdahXDDi1mShAiIsVs4cadDJs0n69Wb6NL81qMujqNLs1rhR3WEVOCEBEpJjv2ZvPw+0sYO30NqVVSeOiS4/hFWnMqlIHmpGiUIEREjlJenvNKxnr++u5itu05wBW9WnLHWe2oVbXsNCdFowQhInIU5q3fzj1vLGDOuu30aFmb5y7oSeemqWGHVSziliDMbDTQH9js7p2jrL8TuCJfHB2B+u6+1cxWA1lALpDj7mnxilNEpCi27T7A395bwgtfraVutUr883+6cEn3ppiVzeakaOJ5BTEGGA48H22lu/8d+DuAmZ0P/Nrdt+bb5DR33xLH+EREjlhunjNx5lr+8f4SsvblMPjE1tx+ZltqVk4JO7RiF7cE4e7TzKxVjJtfDkyMVywiIsUhY802hk2az/wNO+nVug73X9iZ9o1qhB1W3IR+D8LMqgLnALfkK3bgfTNz4Cl3H1nI54cAQwBatGgRz1BFpJzasms/f31nMS9nrKdhzUo8dnk3zj++cUI1J0UTeoIAzgc+P6R56WR332BmDYApZrbY3adF+3CQPEYCpKWlefzDFZHyIic3j7HT1/DwlKXsy87lxlPbcOvpbalWqTScOuOvNNRyAIc0L7n7huDnZjN7HegJRE0QIiLxMGPldwybtIDF32bx07b1GHb+sRzToHrYYZWoUBOEmaUCpwJX5iurBlRw96xg+Szg/pBCFJFyZtPOfTz49iLemLORprWqMOLK7px9bKOEb06KJp6PuU4E+gL1zGw9MAxIAXD3EcFmFwPvu/vufB9tCLweHIxkYIK7vxuvOEVEAA7k5DHmi1U8+sEysnOdX51+DDf1PYYqFZPCDi008XyK6fIYthlD5HHY/GUrgS7xiUpE5L99tmwLwybNZ0Xmbk7v0IB7+neiVb1qYYcVutJwD0JEJBQbtu/lz28t5O2vv6VFnao8MyiNMzo2DDusUkMJQkTKnf05uYz6dBXDP1pOnju/ObMdQ05pQ+WU8tucFI0ShIiUKx8v2cx9kxaw+rs9nH1sQ/5wXiea16kadlilkhKEiJQLa7/bw/1vLuSDRZtoU68az13bk1Pb1Q87rFJNCUJEEtq+7Fye/GQFT05dQXIF47fndOC6k1tTMblC2KGVekoQIpKQ3J0pCzdx/5sLWb9tL/2Pb8zd53WkcWqVsEMrM5QgRCThrNqym3snLWDq0kzaNqjOhBt6ceJP6oUdVpmjBCEiZdr+nFx27M1m594cduzN5sNFmxj16SoqJlfgD+d1ZNCJrUhJUnNSUShBiEiocvOcrH2RE/zOfdnByT4733LkxL9zX6Q8spzz/Xb7c/L+a5+XdGvKXf060KBG5RBqlDiUIETkqLg7e7N/+Ct+575sduz58Qk+/4n/4Al+Z/A+a39OoftPqmDUrJxMzSoppFZJoWblFBqlVv5+uWaV4FU5mdQqKTSrXbXcDaoXL0oQIkJ2bt5/nbx37C34JL8z/0l+XzbZuYWPtF+9UvL3J/maVVJoWqsKHRvX+P4kn3rISf77ZFAlhWoVk8rlQHmlgRKESILal53Lewu+JTNr//cn9IKab/Zm5xa6r4pJFYKTezI1K6dQq2pFWtStRmrwPv9f96n5tkutkkKNyskk6x5AmaQEIZKA9mXncsPz6Xy6LDKtuxnUqJRMatUfTuJt6lWnZpXkH07sVQ822ST/qPkmtUoKlZIr6K/4ckgJQiTBHMjJ4+bxs/h02RYevPg4+ndpTPWKyVSooBO8HBklCJEEkpObx20vzObDxZt54KLODOyledql6NQwKJIgcvOcO16eyzvzv+WP/TtxZe+WYYckZZwShEgCyMtz7np1Hm/M2cj/ndOe605uHXZIkgCUIETKOHfnnknzeTljPbed0Zab+h4TdkiSIJQgRMowd+dPby5i3PS13HhqG27/WduwQ5IEogQhUka5O39/bwmjP1/FNSe24q5zOuhRVClWShAiZdS/P1rOE5+s4PKeLRh2ficlByl2ShAiZdCIqSt4eMpSft69GX++qLOSg8RF3BKEmY02s81mNr+A9Xea2ZzgNd/Mcs2sTrDuHDNbYmbLzeyueMUoUhY9+/kqHnpnMed3acLfLj1eHeAkbuJ5BTEGOKegle7+d3fv6u5dgd8BU919q5klAY8D5wKdgMvNrFMc4xQpMybMWMt9kxdy9rENefgXXUhScpA4iluCcPdpwNYYN78cmBgs9wSWu/tKdz8AvABcGIcQRcqUVzPWc/d/vua09vV57PJumgRH4i70f2FmVpXIlcarQVFTYF2+TdYHZQV9foiZpZtZemZmZvwCFQnR5LkbufOVuZz0k3o8eWUPKiUnhR2SlAOhJwjgfOBzd4/1auNH3H2ku6e5e1r9+vWLOTSR8L07/1tuf3EOaS3rMPLqHlROUXKQklEaEsQAfmheAtgANM/3vllQJlLufLx4M7+aOIvjm6UyevAJVK2o8TWl5ISaIMwsFTgVeCNf8VdAWzNrbWYViSSQSWHEJxKmz5dv4cZxGbRvVIMxg3tSvZKSg5SsuP2LM7OJQF+gnpmtB4YBKQDuPiLY7GLgfXffffBz7p5jZrcA7wFJwGh3XxCvOEVKo5mrtnLdc1/Rpl41xl7bi9QqKWGHJOWQuRc+l2xZkpaW5unp6WGHIXJUZq3dxlWjZtAotTIvDOlD/RqVwg5JEpiZZbh7WrR1peEehIgE5m/YwaDRM6lXoxITbuit5CChUoIQKSUWf7uTK5+ZQc3KKUy4oTcNa1YOOyQp55QgREqB5Zt3ceWoGVROTmLCDb1oWqtK2CGJKEGIhG31lt0MfHo6YIy/oRct61YLOyQR4AgThJlVMLOa8QpGpLxZv20PV4yaQXZuHuOv78VP6lcPOySR7x02QZjZBDOraWbVgPnAQjO7M/6hiSS2b3fsY+DTM8jal83Y63rRvlGNsEMS+ZFYriA6uftO4CLgHaA1cFVcoxJJcJlZ+xk4ajpbdx/g+et60blpatghifyXWBJEipmlEEkQk9w9G0iczhMiJWzr7gNcOWoG32zfx7ODT6Br81phhyQSVSwJ4ilgNVANmGZmLYGd8QxKJFHt2JPNVc/MYPV3u3lmUBontKoTdkgiBTrsUBvu/hjwWL6iNWZ2WvxCEklMWfuyufrZmSzbtIuRV/fgxGPqhR2SSKFiuUl9W3CT2szsGTObBZxeArGJJIw9B3K4dsxXLNiwg+EDu9G3fYOwQxI5rFiamK4NblKfBdQmcoP6obhGJZJA9mXncv1z6WSs2cajA7px1rGNwg5JJCaxJIiDk972A8YGI6tqIlyRGOzPyeXGsRl8ufI7/vmLLpx3fOOwQxKJWSwJIsPM3ieSIN4zsxpAXnzDEin7snPzuGXCbKYuzeQvFx/Hxd2ahR2SyBGJZT6I64CuwEp332NmdYHB8Q1LpGzLyc3j9hfnMGXhJu6/8FgG9GwRdkgiRyyWp5jyzKwZMNDMAKa6++S4RyZSRuXlOf/3yjzemvcNd/fryNV9WoUdkkiRxPIU00PAbcDC4HWrmT0Y78BEyqK8POf3r3/Na7M3cMeZ7bjhlDZhhyRSZLE0MfUDurp7HoCZPQfMBn4fz8BEyhp3577JC3jhq3Xcctox/OqMtmGHJHJUYh3NNf9YABo0RuQQ7s5f3lnMc1+u4YaftuaOs9qFHZLIUYvlCuIvwGwz+5jI462nAHfFNSqRMuZfU5YyctpKru7Tkt/360hwv06kTIvlJvVEM/sEOCEo+i3QMp5BiZQlj3+8nMc+Ws5lac259/xjlRwkYcRyBYG7fwNMOvjezGYCem5Pyr1Rn67k7+8t4eJuTXnwkuOoUEHJQRJHUacc1f8CKffGfrmaB95axHnHNebvlx5PkpKDJJiiJojDzgdhZqPNbLOZzS9km75mNsfMFpjZ1Hzlq83s62BdehFjFImbl75axx/fWMDPOjbkkQFdSU7S9O6SeApsYjKzyURPBAbUjWHfY4DhwPMF7L8W8ARwjruvNbNDh7c8zd23xPA9IiXqP7M38NvX5nFKu/o8fkU3UpQcJEEVdg/iH0VcB4C7TzOzVoVsMhB4zd3XBttvPtw+RcL29tff8JuX5tC7dV2eurIHlZKTwg5JJG4KTBDuPrWgdcWkHZHpTD8BagCPuvvBqw0H3jczB55y95EF7cTMhgBDAFq00H1ziZ8PFm7i1omz6d6iNqMGpVGlopKDJLaYnmKK43f3AM4AqgBfmtl0d18KnOzuG4Jmpylmttjdp0XbSZA8RgKkpaVprmyJi6lLM7lp/CyObVKTZwefQLVKYf7XESkZYTaergfec/fdwb2GaUAXAHffEPzcDLwO9AwtSin3vlixhSHPp3NMg+o8f20valROCTskkRIRZoJ4AzjZzJLNrCrQC1hkZtWCOScws2pEZrIr8EkokXhKX72V659Lp2Xdqoy9riepVZUcpPw47HWymbUD7iTSe/r77d290HmpzWwi0BeoZ2brgWFASvDZEe6+yMzeBeYRmYBolLvPN7M2wOtBb9RkYIK7v1uEuokclbnrtnPNs1/RqGZlxl3fi7rVK4UdkkiJMvfCm+3NbC4wAsgAcg+Wu3tGfEM7cmlpaZ6erm4TcvQWbNzBwKdnULNKMi/d2IfGqVXCDkkkLswsw93Toq2L5U5bjrs/WcwxiZRaSzdlcdUzM6lWMYkJ1/dWcpByK5Z7EJPN7CYza2xmdQ6+4h6ZSAhWZu5i4NMzSK5gTLihN83rVA07JJHQxHIFMSj4eWe+Mgc0VZYklLXf7WHg0zNwdyYM6U2retXCDkkkVLEM9926JAIRCdPG7XsZOGo6+3JymXhDb45pUCPskERCF8tTTCnAUCITBQF8QqR3c3Yc4xIpMZt37mPg09PZsSebCTf0pmPjmmGHJFIqxNLE9CSRx1OfCN5fFZRdH6+gRErKll37GThqBplZ+3n+ul4c10wz6oocFEuCOMHdu+R7/1Hw6KtImbZ9zwGuHDWD9dv28NzgnvRoWTvskERKlVieYso1s58cfBN0ZMstZHuRUm/nvmyuemYmK7fsZtTVJ9CrTSwj2IuUL7FcQdwJfGxmK4nMBdESGBzXqETiaNf+HK4ZPZPF3+7kqat6cHLbemGHJFIqxfIU04dm1hZoHxQtcff98Q1LJD72Hsjl2jFfMXf9Dh4f2J3TOzQMOySRUquwGeVOd/ePzOySQ1YdY2a4+2txjk2kWO3LzmXI2HTSV2/lkQHdOKdzo7BDEinVCruCOBX4CDg/yjoHlCCkzNiXncvN42fx6bIt/ON/unBBlyZhhyRS6hU2o9ywYPF+d1+Vf52ZqfOclBnrtu5h6PgM5m/YyQMXdebSHs3CDkmkTIjlKaZXo5S9UtyBiMTDtKWZnD/8M9Z8t4dnBqVxZe+WYYckUmYUdg+iA3AskHrIfYiaQOV4ByZyNNydJz5ZwT/eX0K7BjV46qoeGltJ5AgVdg+iPdAfqMWP70NkATfEMyiRo5G1L5s7XprL+ws3cUGXJjz08+OoWlFzSIscqcLuQbwBvGFmfdz9yxKMSaTIlm3K4saxGazduod7+ndi8EmtCGYnFJEjFMufVbPN7GYizU3fNy25+7Vxi0qkCN6a9w13vjKXqhWTmXBDb3q21rQlIkcjlpvUY4FGwNnAVKAZkWYmkVIhJzePP7+1kJsnzKJj45q8devJSg4ixSCWK4hj3P1/zOxCd3/OzCYAn8Y7MJFYbNm1n1smzGL6yq0M6tOSu8/rRMXkWP7uEZHDiSVBHJz3YbuZdQa+BRrELySR2Mxau42bxs1i+94D/OuyLlzcTf0bRIpTLAlipJnVBv4ITAKqA/fENSqRQrg742es5b7JC2icWoXXhp5Epyaa5EekuMUyWN+oYHEqRzAPtZmNJvKY7GZ371zANn2BR4hMSLTF3U8Nys8BHgWSgFHu/lCs3yuJbV92Ln/4z3xeyVhP3/b1efSybqRWTQk7LJGEVFhHud8U9kF3f/gw+x4DDAeeL2D/tYjMUneOu681swZBeRLwOHAmsB74yswmufvCw3yfJLj8Q2bcekZbbj+jLRUq6BFWkXgp7Ari4Kzt7YETiDQvQaTT3MzD7djdp5lZq0I2GQi85u5rg+03B+U9geXuvhLAzF4ALgSUIMqxaUszufWF2eTmOc8MSuOMjhqmWyTeCusodx+AmU0Durt7VvD+XuCtYvjudkCKmX1CJBk96u7PA02Bdfm2Ww/0KmgnZjYEGALQokWLYghLSpO8POfJqZEhM9o3rMGIKzVkhkhJieUmdUPgQL73B4Ky4vjuHsAZQBXgSzObfqQ7cfeRwEiAtLQ0L4a4pJTYGQyZMWXhJi7s2oS/XKIhM0RKUiz/254HZprZ68H7i4jcXzha64Hv3H03sDu4UukSlDfPt10zYEMxfJ+UIUs3ZfHLYMiMYed34poTNWSGSEmL5SmmP5vZO8BPg6LB7j67GL77DWC4mSUDFYk0I/0LWAy0Deac2AAMIHK/QsqJN+dt5P9emUe1SslMHNKbE1qpV7RIGAp7iqmmu+80szrA6uB1cF0dd99a2I7NbCLQF6hnZuuBYUQeZ8XdR7j7IjN7F5gH5BF5nHV+8NlbgPeIPOY62t0XFLmGUmbk5Obx0DuLGfXZKtJa1uaJK7rToKZGlhcJi7lHb7Y3szfdvb+ZrSIyxej3qwB395j7RJSUtLQ0T09PDzsMKYLMrMiQGTNWbeWaE1vx+34dNWSGSAkwswx3T4u2rrCnmPoHPzW9qMSVhswQKZ0Ka2LqXtgH3X1W8Ycj5Ym7M27GWu7XkBkipVJhN6n/Wcg6B04v5likHNmXncvdr8/n1VnrOa19fR7RkBkipU5hTUynlWQgUn6s27qHX47LYMHGndx2Rltu05AZIqVSTL2OgmG+O/HjGeWijrEkUpiDQ2bk5Tmjr0nj9A4aMkOktDpsgjCzYUQeV+0EvA2cC3xGAYPwiUSTl+c88cly/jllKe0b1uCpq3rQsq6GzBApzWK5griUSA/n2e4+2MwaAuPiG5YkkvxDZlzUtQl/ueR4qlRMCjssETmMWBLEXnfPM7McM6sJbObHQ2GIFGjJt1n8clwG67bu4d7zOzFIQ2aIlBmxJIj0YO6Gp4EMYBfwZVyjkoQweW5kyIzqlTVkhkhZVFg/iMeBCe5+U1A0Ihgao6a7zyuR6KRMyg6GzHhGQ2aIlGmFXUEsBf5hZo2Bl4CJxTRInyQwDZkhkjgK6wfxKPCombUkMqLqaDOrAkwkkiyWllCMUkbMWruNoeMy2LE3W0NmiCSAw/5p5+5r3P2v7t4NuJzIfBCL4h6ZlBnuztjpa7jsqS+plJzEa0NPUnIQSQCx9INIJtL3YQCR2d8+Ae6Na1RSZmjIDJHEVdhN6jOJXDH0A2YCLwBDghngRL4fMmPhNzu5/WdtufV0DZkhkkgKu4L4HTABuMPdt5VQPFJGTF2ayW0Hh8wYdAKndWgQdkgiUswKu0mt0Vrlv2jIDJHyI6bB+kQgMmTGb16cyweLNnFxt6Y8ePFxGjJDJIEpQUhM8g+Zcd8Fx3J1n5YaMkMkwSlByGFpyAyR8kkJQgqUf8iME1rV5vGBGjJDpDxRgpCoMrP2c/OEWcwMhsy4+7yOpCRpyAyR8iRuCcLMRgP9gc3u3jnK+r7AG8CqoOg1d78/WLcayAJygRx3T4tXnPLfMtZs46bxkSEzHrmsKxd1axp2SCISgnheQYwBhlP4zHOfunv/Atad5u5bij0qKZC7M27GWu6fvIAmtaowZnBPOjauGXZYIhKSuCUId59mZq3itX8pXvmHzDi9QwP+9YuuGjJDpJwLu1G5j5nNNbN3zOzYfOUOvG9mGWY2pLAdmNkQM0s3s/TMzMz4Rpug1m3dw8+f/ILXZq/n1z9rx6ir05QcRCTUm9SzgJbuvsvM+gH/AdoG60529w1m1gCYYmaL3X1atJ24+0hgJEBaWpqXROCJZOrSTG6dOBt3DZkhIj8W2hWEu+90913B8ttAipnVC95vCH5uBl4HeoYVZ6LKy3P+/eEyrnl2Jo1TKzP5VycrOYjIj4R2BWFmjYBN7u5m1pNIsvrOzKoBFdw9K1g+C7g/rDgT0Y692dzxkobMEJHCxfMx14lAX6Cema0HhgEpAO4+ArgUGGpmOcBeYECQLBoCrwfDOCQTmRf73XjFWd7M37CDm8bPYuP2vdx7ficGndhKQ2aISFTxfIrp8sOsH07kMdhDy1cCXeIVV3n24ldr+eMbC6hbrSIv3tiHHi1rhx2SiJRi6kldDuw9kMs9b8zn5Yz1/LRtPR65rCt1q1cKOywRKeWUIBLcqi27GTougyWbsrj1jLbcdkZbkjTrm4jEQAkigb07/xvufHkeSUnGs9ecQN/2ekpJRGKnBJGAsnPz+Nu7i3n601V0aZbK41d0p1ntqmGHJSJljBJEgtm0cx+3TJjFV6u3cXWfltx9XkcqJesRVhE5ckoQCeSLFVu4deJsdu/P5dEBXbmwq0ZhFZGiU4JIAHl5zpNTV/DP95fQul41Jt7Qm7YNa4QdloiUcUoQZdyOPdn85qU5fLh4M/2Pb8xDPz+e6pV0WEXk6OlMUobN37CDoeMz+HbHPu674Fiu7tNSvaJFpNgoQZRB7s7Emeu4d/IC6lWryEs39qFbC/WKFpHipQRRxuw9kMvd//ma12Zt4JR29Xnksq7UqVYx7LBEJAEpQZQhKzN3cdP4WSzZlMXtP2vLr05Xr2gRiR8liDLina+/4c5X5pGSZIwZ3JNT29UPOyQRSXBKEKVcdm4eD72zmGc+W0XX5rV44oruNKlVJeywRKQcUIIoxb7dEekVnb5mG9ec2Irf9+tIxeSwpxEXkfJCCaKU+nx5pFf03uxcHru8Gxd0aRJ2SCJSzihBlDJ5ec4Tnyzn4SlLaVO/Oi9e2Z1jGqhXtIiUPCWIUmT7ngP8+sU5fLwkkwu7NuHBi4+jmnpFi0hIdPYpJeat387QcbPYnLWPP114LFf2Vq9oEQmXEkTI3J3xM9Zy/+SF1K9RiZd/eSJdm9cKOywRESWIMO05kMPdr8/n9dkbODXoFV1bvaJFpJRQggjJisxdDB2XwbLNu/jNme245bRjqKBe0SJSisTtoXozG21mm81sfgHr+5rZDjObE7zuybfuHDNbYmbLzeyueMUYlrfmfcMF//6MLbsO8Py1Pbn1jLZKDiJS6sTzCmIMMBx4vpBtPnX3/vkLzCwJeBw4E1gPfGVmk9x9YbwCLSkHcvL4yzuLePbz1XRrUYvHB6pXtIiUXnFLEO4+zcxaFeGjPYHl7r4SwMxeAC4EynSC+GbHXm4eP4tZa7cz+KRW/O5c9YoWkdIt7HsQfcxsLrAR+F93XwA0Bdbl22Y90KugHZjZEGAIQIsWLeIYatF9tmwLt74wm/3ZuQwf2I3+x6tXtIiUfmEmiFlAS3ffZWb9gP8AbY90J+4+EhgJkJaW5sUb4tHJy3OGf7ycf32wlLYNqvPEFT04pkH1sMMSEYlJaAnC3XfmW37bzJ4ws3rABqB5vk2bBWVlyrbdB/j1S3P4ZEkmF3VtwoOXHEfVimFfsImIxC60M5aZNQI2ububWSlx0yQAAAiRSURBVE8iT1R9B2wH2ppZayKJYQAwMKw4i2LOuu3cPH4WmVn7eeCizlzRq4V6RYtImRO3BGFmE4G+QD0zWw8MA1IA3H0EcCkw1MxygL3AAHd3IMfMbgHeA5KA0cG9iVLP3Rk3fQ33v7mQBjUq8/Iv+9BFvaJFpIyyyDk5MaSlpXl6enoo3717fw6/f/1r3pizkb7t6/OvX6hXtIiUfmaW4e5p0dapUbwYLN+cxdBxs1ieuYs7zmzHzeoVLSIJQAniKE2eu5HfvjqPKilJjL22Fye3rRd2SCIixUIJoogO5OTx4NuLGPPFanq0rM3wgd1onKpe0SKSOJQgimDj9r3cNH4Wc9Zt59qTWvO7fh1ISVKvaBFJLEoQR2ja0kxue2E22bnOE1d0p99xjcMOSUQkLpQgYpSX5zz20TIe/XAZ7RrU4Mkru9OmvnpFi0jiUoKIwdbdB7j9xTlMW5rJJd2a8sDFndUrWkQSns5yhzF77TZuHj+LLbsO8ODFx3F5z+bqFS0i5YISRAHcnee/XMMDby2kYc3KvDr0RI5rlhp2WCIiJUYJIord+3O467WvmTx3I6d3aMDDv+hCrarqFS0i5YsSxCGWb87il+NmsTJzF3ee3Z6hp/5EvaJFpFxSgsjnjTkb+N1rX1O1YhLjruvFiceoV7SIlF9KEMD+nFweeHMRY6evIa1lbYYP7E6j1MphhyUiEqpynyB27Mnm6mdnMnfddq4/uTW/PVe9okVEQAmCGpWTaVW3Kr88pQ3nqle0iMj3yn2CqFDBeHRAt7DDEBEpddSWIiIiUSlBiIhIVEoQIiISlRKEiIhEpQQhIiJRKUGIiEhUShAiIhKVEoSIiERl7h52DMXGzDKBNUX8eD1gSzGGUxaozomvvNUXVOcj1dLd60dbkVAJ4miYWbq7p4UdR0lSnRNfeasvqM7FSU1MIiISlRKEiIhEpQTxg5FhBxAC1Tnxlbf6gupcbHQPQkREotIVhIiIRKUEISIiUZWbBGFmq83sazObY2bpQVkdM5tiZsuCn7WDcjOzx8xsuZnNM7Pu4UYfGzMbbWabzWx+vrIjrqOZDQq2X2Zmg8KoS6wKqPO9ZrYhONZzzKxfvnW/C+q8xMzOzld+TlC23MzuKul6HAkza25mH5vZQjNbYGa3BeUJeawLqW/CHmczq2xmM81sblDn+4Ly1mY2I4j/RTOrGJRXCt4vD9a3yrevqL+LmLh7uXgBq4F6h5T9DbgrWL4L+Guw3A94BzCgNzAj7PhjrOMpQHdgflHrCNQBVgY/awfLtcOu2xHW+V7gf6Ns2wmYC1QCWgMrgKTgtQJoA1QMtukUdt0KqXNjoHuwXANYGtQtIY91IfVN2OMcHKvqwXIKMCM4di8BA4LyEcDQYPkmYESwPAB4sbDfRaxxlJsriAJcCDwXLD8HXJSv/HmPmA7UMrNSP2G1u08Dth5SfKR1PBuY4u5b3X0bMAU4J/7RF00BdS7IhcAL7r7f3VcBy4GewWu5u6909wPAC8G2pZK7f+Pus4LlLGAR0JQEPdaF1LcgZf44B8dqV/A2JXg5cDrwSlB+6DE+eOxfAc4wM6Pg30VMylOCcOB9M8swsyFBWUN3/yZY/hZoGCw3Bdbl++x6Cv8HWZodaR0Tpe63BM0pow82tZCAdQ6aEroR+Qsz4Y/1IfWFBD7OZpZkZnOAzUSS9wpgu7vnBJvkj//7ugXrdwB1Oco6l6cEcbK7dwfOBW42s1Pyr/TI9VhCP/NbHuoYeBL4CdAV+Ab4Z7jhxIeZVQdeBW5395351yXisY5S34Q+zu6e6+5dgWZE/urvUNIxlJsE4e4bgp+bgdeJ/MI3HWw6Cn5uDjbfADTP9/FmQVlZdKR1LPN1d/dNwX+uPOBpfrikTpg6m1kKkZPleHd/LShO2GMdrb7l4TgDuPt24GOgD5HmweRgVf74v69bsD4V+I6jrHO5SBBmVs3MahxcBs4C5gOTgINPbgwC3giWJwFXB09/9AZ25Lt0L2uOtI7vAWeZWe3gkv2soKzMOOR+0cVEjjVE6jwgeOKjNdAWmAl8BbQNnhCpSOQm36SSjPlIBG3LzwCL3P3hfKsS8lgXVN9EPs5mVt/MagXLVYAzidx7+Ri4NNjs0GN88NhfCnwUXEUW9LuITdh360viReSphbnBawFwd1BeF/gQWAZ8ANTxH54geJxIm9/XQFrYdYixnhOJXGpnE2lrvK4odQSuJXIzazkwOOx6FaHOY4M6zQv+gzTOt/3dQZ2XAOfmK+9H5OmYFQf/fZTWF3AykeajecCc4NUvUY91IfVN2OMMHA/MDuo2H7gnKG9D5AS/HHgZqBSUVw7eLw/Wtznc7yKWl4baEBGRqMpFE5OIiBw5JQgREYlKCUJERKJSghARkaiUIEREJColCJGjYGZ3B6NtzgtGFO1lZrebWdWwYxM5WnrMVaSIzKwP8DDQ1933m1k9IqOEfkGkr8GWUAMUOUq6ghApusbAFnffDxAkhEuBJsDHZvYxgJmdZWZfmtksM3s5GFPo4Bwlf7PIPCUzzeyYsCoiEo0ShEjRvQ80N7OlZvaEmZ3q7o8BG4HT3P204KriD8DPPDJYZDrwm3z72OHuxwHDgUdKugIihUk+/CYiEo277zKzHsBPgdOAF6PMUtabyKQtn0eGFKIi8GW+9RPz/fxXfCMWOTJKECJHwd1zgU+AT8zsa34YMO0gIzIpz+UF7aKAZZHQqYlJpIjMrL2Ztc1X1BVYA2QRmRoTYDpw0sH7C8HIwu3yfeayfD/zX1mIhE5XECJFVx34dzAscw6RkTSHAJcD75rZxuA+xDXARDOrFHzuD0RGFAWobWbzgP3B50RKDT3mKhISM1uNHoeVUkxNTCIiEpWuIEREJCpdQYiISFRKECIiEpUShIiIRKUEISIiUSlBiIhIVP8P+lI0PrqHP7AAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "steps=[500,1000 ,1500 ,2000 ,2500 ,3000 ]\n",
        "loss= [1.467952,1.543128 , 1.619953,1.721062 , 1.727321, 1.785547 ]\n",
        "plt.plot(steps, loss)\n",
        "plt.xlabel(\"Step\")\n",
        "plt.ylabel(\"Validation Loss\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        },
        "id": "SyFVSL_Ops2F",
        "outputId": "9ee6bc7a-b878-4cd0-da44-202353c68b3e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 5000\n",
            "  Batch size = 8\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='625' max='625' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [625/625 01:14]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "{'epoch': 5.0,\n",
              " 'eval_f1': 0.5385933649075498,\n",
              " 'eval_loss': 1.7877246141433716,\n",
              " 'eval_runtime': 75.1251,\n",
              " 'eval_samples_per_second': 66.556,\n",
              " 'eval_steps_per_second': 8.319}"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "evaluation_result = trainer.evaluate()\n",
        "evaluation_result"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w4HbhwQb5Zi_"
      },
      "source": [
        "### get the masked word prediction (sample sentences above) on the fine-tuned model, why the results as they are and what should be done in order to change that (write down your answer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6qsc65dr5cpB",
        "outputId": "d7d8a44c-bfbc-4c82-cd4d-44d4471c42da"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Configuration saved in /content/drive/MyDrive/Untitled Folder/trained/config.json\n",
            "Model weights saved in /content/drive/MyDrive/Untitled Folder/trained/pytorch_model.bin\n"
          ]
        }
      ],
      "source": [
        "model.save_pretrained(\"/content/drive/MyDrive/Untitled Folder/trained\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aahCxJ2LqWYK",
        "outputId": "04c8a837-8aaf-48d3-9683-3cb7d5303bee"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file /content/drive/MyDrive/Untitled Folder/trained/config.json\n",
            "Model config ElectraConfig {\n",
            "  \"_name_or_path\": \"google/electra-small-generator\",\n",
            "  \"architectures\": [\n",
            "    \"ElectraForSequenceClassification\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"embedding_size\": 128,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 256,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 1024,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"electra\",\n",
            "  \"num_attention_heads\": 4,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"problem_type\": \"single_label_classification\",\n",
            "  \"summary_activation\": \"gelu\",\n",
            "  \"summary_last_dropout\": 0.1,\n",
            "  \"summary_type\": \"first\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"torch_dtype\": \"float32\",\n",
            "  \"transformers_version\": \"4.17.0\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "loading weights file /content/drive/MyDrive/Untitled Folder/trained/pytorch_model.bin\n",
            "Some weights of the model checkpoint at /content/drive/MyDrive/Untitled Folder/trained were not used when initializing ElectraForMaskedLM: ['classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.out_proj.weight']\n",
            "- This IS expected if you are initializing ElectraForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing ElectraForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of ElectraForMaskedLM were not initialized from the model checkpoint at /content/drive/MyDrive/Untitled Folder/trained and are newly initialized: ['generator_predictions.LayerNorm.bias', 'generator_predictions.dense.weight', 'generator_lm_head.weight', 'generator_predictions.dense.bias', 'generator_predictions.LayerNorm.weight', 'generator_lm_head.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "trained_model = ElectraForMaskedLM.from_pretrained(\"/content/drive/MyDrive/Untitled Folder/trained\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P2kpKKoUrOe3"
      },
      "outputs": [],
      "source": [
        "predictor_trained = pipeline(\"fill-mask\", model=trained_model, tokenizer=tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "asE46eSqsTdp",
        "outputId": "5fcd33ba-5cea-435a-fe60-205991a5fb24"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.0003825551248155534,\n",
              "  'sequence': \"why don't you asklizer?\",\n",
              "  'token': 28863,\n",
              "  'token_str': '# # l i z e r'},\n",
              " {'score': 0.00036645226646214724,\n",
              "  'sequence': \"why don't you ask manitoba?\",\n",
              "  'token': 10512,\n",
              "  'token_str': 'm a n i t o b a'},\n",
              " {'score': 0.0003258672368247062,\n",
              "  'sequence': \"why don't you askhale?\",\n",
              "  'token': 15238,\n",
              "  'token_str': '# # h a l e'},\n",
              " {'score': 0.00030835444340482354,\n",
              "  'sequence': \"why don't you ask prime?\",\n",
              "  'token': 3539,\n",
              "  'token_str': 'p r i m e'},\n",
              " {'score': 0.0003023090248461813,\n",
              "  'sequence': \"why don't you askerland?\",\n",
              "  'token': 22492,\n",
              "  'token_str': '# # e r l a n d'}]"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictor_trained(sen_1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tsyUm88Wshxh",
        "outputId": "5c9f15d3-9adc-4d47-d4aa-33b3dbe96ab6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.0003889001382049173,\n",
              "  'sequence': 'what islizer?',\n",
              "  'token': 28863,\n",
              "  'token_str': '# # l i z e r'},\n",
              " {'score': 0.0003770088078454137,\n",
              "  'sequence': 'what iswest?',\n",
              "  'token': 19650,\n",
              "  'token_str': '# # w e s t'},\n",
              " {'score': 0.0003740653919521719,\n",
              "  'sequence': 'what islington?',\n",
              "  'token': 18722,\n",
              "  'token_str': '# # l i n g t o n'},\n",
              " {'score': 0.0003371702623553574,\n",
              "  'sequence': 'what is chihuahua?',\n",
              "  'token': 28480,\n",
              "  'token_str': 'c h i h u a h u a'},\n",
              " {'score': 0.00031758195837028325,\n",
              "  'sequence': 'what isguide?',\n",
              "  'token': 28582,\n",
              "  'token_str': '# # g u i d e'}]"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictor_trained(sen_2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "02iimMLEsiG4",
        "outputId": "3d8eaf1b-6d11-4822-99e4-2571ad57a1a2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[{'score': 0.00028710137121379375,\n",
              "  'sequence': \"let's talk aboutnded physics\",\n",
              "  'token': 25848,\n",
              "  'token_str': '# # n d e d'},\n",
              " {'score': 0.0002739493502303958,\n",
              "  'sequence': \"let's talk about bhp physics\",\n",
              "  'token': 22245,\n",
              "  'token_str': 'b h p'},\n",
              " {'score': 0.0002729956468101591,\n",
              "  'sequence': \"let's talk aboutcian physics\",\n",
              "  'token': 14483,\n",
              "  'token_str': '# # c i a n'},\n",
              " {'score': 0.00026779426843859255,\n",
              "  'sequence': \"let's talk about cerro physics\",\n",
              "  'token': 25498,\n",
              "  'token_str': 'c e r r o'},\n",
              " {'score': 0.00026544061256572604,\n",
              "  'sequence': \"let's talk about frigate physics\",\n",
              "  'token': 15437,\n",
              "  'token_str': 'f r i g a t e'}]"
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predictor_trained(sen_3)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gruq0jWr5d-e"
      },
      "source": [
        "### Tune the training hyperparameters (and write down your results)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "u6yfLS2kpoJY"
      },
      "outputs": [],
      "source": [
        "device = 'cuda'\n",
        "import torch, gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hXBpvrTloRMJ",
        "outputId": "bbde6cfd-e576-4557-9aab-9b4557551197"
      },
      "outputs": [],
      "source": [
        "#tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')\n",
        "#dataset = load_dataset('glue', 'mrpc')\n",
        "#metric = load_metric('glue', 'mrpc')\n",
        "\n",
        "\n",
        "#encoded_dataset = dataset.map(encode, batched=True)\n",
        "# Won't be necessary when this PR is merged with master since the Trainer will do it automatically\n",
        "#encoded_dataset.set_format(columns=['attention_mask', 'input_ids', 'token_type_ids', 'label'])\n",
        "\n",
        "def model_init():\n",
        "    return ElectraForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=10, return_dict=True)\n",
        "\n",
        "#def compute_metrics(eval_pred):\n",
        "#    predictions, labels = eval_pred\n",
        "#    predictions = predictions.argmax(axis=-1)\n",
        "#    return metric.compute(predictions=predictions, references=labels)\n",
        "\n",
        "# Evaluate during training and a bit more often than the default to be able to prune bad trials early.\n",
        "# Disabling tqdm is a matter of preference.\n",
        "#training_args = TrainingArguments(\"test\", evaluate_during_training=True, eval_steps=500, disable_tqdm=True)\n",
        "trainer = Trainer(\n",
        "    args=training_args,\n",
        "    #data_collator=DataCollatorWithPadding(tokenizer),\n",
        "    train_dataset=training, #encoded_dataset[\"train\"], \n",
        "    eval_dataset=evaluating, #encoded_dataset[\"validation\"], \n",
        "    model_init=model_init,\n",
        "    compute_metrics=compute_metrics,\n",
        ")\n",
        "\n",
        "# Defaut objective is the sum of all metrics when metrics are provided, so we have to maximize it.\n",
        "trainer.hyperparameter_search(direction=\"maximize\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DpGJufBJciil",
        "outputId": "f2bbad74-b8da-4911-a99a-bb536a10f5cc",
        "pycharm": {
          "name": "#%%\n"
        }
      },
      "outputs": [],
      "source": [
        "trainer.evaluate()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XETNkrqTeW2y"
      },
      "source": [
        "f1 before tuning 0.54 _____ f1 after tuning 0.58 _____ difference 8%"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "[Nemat_Aloush_J41332c]HW4.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "002b55b925e840f0ad9cf491dce2b2c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "064ed687126b4a548c69349444213dd4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "065cae0e684449ba82815ebd39e48a08": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f2b849756eef4972bbe0309acc93e71a",
            "placeholder": "​",
            "style": "IPY_MODEL_8fa0f89f3ddd486aa984dce1dd3b8b33",
            "value": " 3.60k/? [00:00&lt;00:00, 79.3kB/s]"
          }
        },
        "093875ee0cde4df3bc90a5703bfa32ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a50d6967b49c465e92a9c85c53dd45eb",
              "IPY_MODEL_63e548f2359f469285dc9a7000f89082",
              "IPY_MODEL_80208d26cd274c768843ba736c911a0c"
            ],
            "layout": "IPY_MODEL_fc095f16bef443fbb2bac3848b195c9d"
          }
        },
        "0fb38cd3a8fe49fc892b1ebd42c61685": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ee8ac67bba6845dd80f02b72c29b15e9",
            "placeholder": "​",
            "style": "IPY_MODEL_9319afc8f34a42048f72db5abc0a4511",
            "value": " 2/2 [00:00&lt;00:00, 43.90it/s]"
          }
        },
        "11680c21182040a39ea564dab32cb243": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7af503ff1d884980a71a97dabeb3b86c",
            "max": 60000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dd85342f892849caaa0cfedd96a19372",
            "value": 60000
          }
        },
        "1d415f6f20834c7eb570cbbf458ffa51": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d67f22868504b56b2d79dcb0465a239": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1ee0f62fe7dc426fb9fd0f37cdbec8b1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2068e394fe364942b73b71dcee3b4a9f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "242a023e4ee14159885cc486161cd9a6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "269a8413edaf4d69bc739148394901e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfd798a5ec7e47ee8b00133ce807c1eb",
            "placeholder": "​",
            "style": "IPY_MODEL_1d67f22868504b56b2d79dcb0465a239",
            "value": "Generating train split: 100%"
          }
        },
        "378c853b3d1445a388af190424b7483b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c63a6d0261f46e3a64b09b6b0120619": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ee0f62fe7dc426fb9fd0f37cdbec8b1",
            "placeholder": "​",
            "style": "IPY_MODEL_ac69acbb66b84e7fb249780927ee525c",
            "value": "Downloading metadata: "
          }
        },
        "46195adf6bd94a249ab04df6878c54b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f536b30b499246d4932eaeb9abdfcfaf",
              "IPY_MODEL_11680c21182040a39ea564dab32cb243",
              "IPY_MODEL_a44957c1e08f4c9887885593950ce557"
            ],
            "layout": "IPY_MODEL_9416f30f744249de93276c9ed1fa8709"
          }
        },
        "46760e479a3642f78015d64dfbacc770": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fac559c605094bf9980ad563c8bc8665",
            "max": 319476345,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ed2ed3dd67d3437da5837778eb6cb823",
            "value": 319476345
          }
        },
        "4689ed3126874788bf1ec1aeabc689c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e8808b6ecc6b49aeb896aa505f50b831",
              "IPY_MODEL_a0c7aa324ee54c65b2e8e099d1ccc212",
              "IPY_MODEL_065cae0e684449ba82815ebd39e48a08"
            ],
            "layout": "IPY_MODEL_fee97eaaa80b4794bc4ec2f4fbb6524d"
          }
        },
        "4d504996b8d84dc8897a763c596eccec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d09b0b11518f497eb053fb3906bc5cf4",
              "IPY_MODEL_46760e479a3642f78015d64dfbacc770",
              "IPY_MODEL_b5eac42a7ee14b0b9f7def444d96f1e6"
            ],
            "layout": "IPY_MODEL_8d0e619ed0774a41abdb6a9ef5db7a96"
          }
        },
        "4dd4b83d9bc24e528c4dbfaff10a7851": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7fd3d69be9794181bffd685996f86c62",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7cfe75a2932a4eb4b41702250f6a93a6",
            "value": 2
          }
        },
        "538f787428684cdbb9f9d94a005de425": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5539451cc1624d00b49e11e0f9df713a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5bbc267f0cfb4ea0bf2dc59076653509": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a5e4167122054541870a76dd2dae4dd6",
            "max": 847,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d15dfb35ee96428690b888d8cfa0e164",
            "value": 847
          }
        },
        "63e548f2359f469285dc9a7000f89082": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a5e5eac91d3148f99bbf0572801025fd",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_db9485e4c39248449317b843c49c134b",
            "value": 466062
          }
        },
        "677bf3bd8c564fc280065562aca5a152": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6debe5a643f04fa7808bc3e371f23cf6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71ed4d5d2f604a1484a4b2d9cbe4c663": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0ee47f5f594486aa30a8b0fd0ac85d7",
            "placeholder": "​",
            "style": "IPY_MODEL_538f787428684cdbb9f9d94a005de425",
            "value": "100%"
          }
        },
        "75b2f07b56bb482fa3f3212e8efc628f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77b9a4c95c5d4a449bef373fbfb545d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7af503ff1d884980a71a97dabeb3b86c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7cfe75a2932a4eb4b41702250f6a93a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7f12a6ba25fb4a9b9b96c03ef0fd4bb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_269a8413edaf4d69bc739148394901e4",
              "IPY_MODEL_bdf0a50f3ced4aedb30613ca14a02648",
              "IPY_MODEL_bd658fa357b548819ada046f2206773b"
            ],
            "layout": "IPY_MODEL_d7d77babe2764ce98750114c12920943"
          }
        },
        "7f89dae5d2564c828fdaba0a2a6fd684": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7fd3d69be9794181bffd685996f86c62": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80208d26cd274c768843ba736c911a0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8b39cafe63324788afaf2922dab983f0",
            "placeholder": "​",
            "style": "IPY_MODEL_c9695739961f43adbebf7c88bc32fc6c",
            "value": " 455k/455k [00:00&lt;00:00, 1.02MB/s]"
          }
        },
        "823c8a14a0564c43835bfd8690f707a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ea79830b6b8444c688bf6e8c889e1ded",
            "placeholder": "​",
            "style": "IPY_MODEL_a9b3b30457c2480dbe6ff826acb6cbce",
            "value": " 1.88k/? [00:00&lt;00:00, 39.3kB/s]"
          }
        },
        "8b39cafe63324788afaf2922dab983f0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d0e619ed0774a41abdb6a9ef5db7a96": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fa0f89f3ddd486aa984dce1dd3b8b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "915b4f3170dd4c33b7016d08c418fe2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9319afc8f34a42048f72db5abc0a4511": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9416f30f744249de93276c9ed1fa8709": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "952f0dfbfdbc48aaadd541248370d918": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3c63a6d0261f46e3a64b09b6b0120619",
              "IPY_MODEL_5bbc267f0cfb4ea0bf2dc59076653509",
              "IPY_MODEL_823c8a14a0564c43835bfd8690f707a5"
            ],
            "layout": "IPY_MODEL_c96c1a2ff4d04ace9f9bc1f76547b538"
          }
        },
        "991e577f5c054278952fb166781d302c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a0c7aa324ee54c65b2e8e099d1ccc212": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_064ed687126b4a548c69349444213dd4",
            "max": 1400,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_002b55b925e840f0ad9cf491dce2b2c3",
            "value": 1400
          }
        },
        "a44957c1e08f4c9887885593950ce557": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec1eb5a0b28046b0bf7bc146382c6f4f",
            "placeholder": "​",
            "style": "IPY_MODEL_677bf3bd8c564fc280065562aca5a152",
            "value": " 59924/60000 [00:10&lt;00:00, 12845.59 examples/s]"
          }
        },
        "a50d6967b49c465e92a9c85c53dd45eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2068e394fe364942b73b71dcee3b4a9f",
            "placeholder": "​",
            "style": "IPY_MODEL_e4b7aff9c8634f04989e0b01362d3705",
            "value": "Downloading: 100%"
          }
        },
        "a5e4167122054541870a76dd2dae4dd6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5e5eac91d3148f99bbf0572801025fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9b3b30457c2480dbe6ff826acb6cbce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ac69acbb66b84e7fb249780927ee525c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b5eac42a7ee14b0b9f7def444d96f1e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5539451cc1624d00b49e11e0f9df713a",
            "placeholder": "​",
            "style": "IPY_MODEL_915b4f3170dd4c33b7016d08c418fe2e",
            "value": " 319M/319M [00:11&lt;00:00, 30.8MB/s]"
          }
        },
        "bcf7663a5f0b48889718932a515f8b22": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd658fa357b548819ada046f2206773b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f89dae5d2564c828fdaba0a2a6fd684",
            "placeholder": "​",
            "style": "IPY_MODEL_1d415f6f20834c7eb570cbbf458ffa51",
            "value": " 1399360/1400000 [01:51&lt;00:00, 13248.84 examples/s]"
          }
        },
        "bdf0a50f3ced4aedb30613ca14a02648": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dad2bef78f98433381db964d767c639d",
            "max": 1400000,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_991e577f5c054278952fb166781d302c",
            "value": 1400000
          }
        },
        "c837d4417e7a4342ad1faa6e6d97f45f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_71ed4d5d2f604a1484a4b2d9cbe4c663",
              "IPY_MODEL_4dd4b83d9bc24e528c4dbfaff10a7851",
              "IPY_MODEL_0fb38cd3a8fe49fc892b1ebd42c61685"
            ],
            "layout": "IPY_MODEL_6debe5a643f04fa7808bc3e371f23cf6"
          }
        },
        "c9695739961f43adbebf7c88bc32fc6c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c96c1a2ff4d04ace9f9bc1f76547b538": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d09b0b11518f497eb053fb3906bc5cf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_75b2f07b56bb482fa3f3212e8efc628f",
            "placeholder": "​",
            "style": "IPY_MODEL_77b9a4c95c5d4a449bef373fbfb545d2",
            "value": "Downloading data: 100%"
          }
        },
        "d15dfb35ee96428690b888d8cfa0e164": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d7d77babe2764ce98750114c12920943": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dad2bef78f98433381db964d767c639d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db9485e4c39248449317b843c49c134b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dd85342f892849caaa0cfedd96a19372": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "dfd798a5ec7e47ee8b00133ce807c1eb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0ee47f5f594486aa30a8b0fd0ac85d7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4b7aff9c8634f04989e0b01362d3705": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e8808b6ecc6b49aeb896aa505f50b831": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_242a023e4ee14159885cc486161cd9a6",
            "placeholder": "​",
            "style": "IPY_MODEL_ea016fe4774449819020c64873ae42ae",
            "value": "Downloading builder script: "
          }
        },
        "ea016fe4774449819020c64873ae42ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ea79830b6b8444c688bf6e8c889e1ded": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec1eb5a0b28046b0bf7bc146382c6f4f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed2ed3dd67d3437da5837778eb6cb823": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ee8ac67bba6845dd80f02b72c29b15e9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2b849756eef4972bbe0309acc93e71a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f536b30b499246d4932eaeb9abdfcfaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_378c853b3d1445a388af190424b7483b",
            "placeholder": "​",
            "style": "IPY_MODEL_bcf7663a5f0b48889718932a515f8b22",
            "value": "Generating test split: 100%"
          }
        },
        "fac559c605094bf9980ad563c8bc8665": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc095f16bef443fbb2bac3848b195c9d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fee97eaaa80b4794bc4ec2f4fbb6524d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}