large-language-model-finetuned.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d9a97912",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:15:30.229520Z",
     "iopub.status.busy": "2023-08-09T06:15:30.229007Z",
     "iopub.status.idle": "2023-08-09T06:17:35.721320Z",
     "shell.execute_reply": "2023-08-09T06:17:35.720059Z"
    },
    "papermill": {
     "duration": 125.503432,
     "end_time": "2023-08-09T06:17:35.723908",
     "exception": false,
     "start_time": "2023-08-09T06:15:30.220476",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing /kaggle/input/llm-package/LLM_Package/accelerate-0.21.0-py3-none-any.whl\r\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (1.23.5)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (21.3)\r\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (5.9.3)\r\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (6.0)\r\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate==0.21.0) (2.0.0)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate==0.21.0) (3.0.9)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (3.12.2)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (4.6.3)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (3.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate==0.21.0) (3.1.2)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate==0.21.0) (2.1.3)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate==0.21.0) (1.3.0)\r\n",
      "Installing collected packages: accelerate\r\n",
      "  Attempting uninstall: accelerate\r\n",
      "    Found existing installation: accelerate 0.20.3\r\n",
      "    Uninstalling accelerate-0.20.3:\r\n",
      "      Successfully uninstalled accelerate-0.20.3\r\n",
      "Successfully installed accelerate-0.21.0\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/bitsandbytes-0.41.1-py3-none-any.whl\r\n",
      "Installing collected packages: bitsandbytes\r\n",
      "Successfully installed bitsandbytes-0.41.1\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/einops-0.6.1-py3-none-any.whl\r\n",
      "Installing collected packages: einops\r\n",
      "Successfully installed einops-0.6.1\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/langsmith-0.0.19-py3-none-any.whl\r\n",
      "Requirement already satisfied: pydantic<2,>=1 in /opt/conda/lib/python3.10/site-packages (from langsmith==0.0.19) (1.10.10)\r\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/conda/lib/python3.10/site-packages (from langsmith==0.0.19) (2.31.0)\r\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.10/site-packages (from pydantic<2,>=1->langsmith==0.0.19) (4.6.3)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langsmith==0.0.19) (3.1.0)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langsmith==0.0.19) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langsmith==0.0.19) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langsmith==0.0.19) (2023.5.7)\r\n",
      "Installing collected packages: langsmith\r\n",
      "Successfully installed langsmith-0.0.19\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/openapi_schema_pydantic-1.2.4-py3-none-any.whl\r\n",
      "Requirement already satisfied: pydantic>=1.8.2 in /opt/conda/lib/python3.10/site-packages (from openapi-schema-pydantic==1.2.4) (1.10.10)\r\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.10/site-packages (from pydantic>=1.8.2->openapi-schema-pydantic==1.2.4) (4.6.3)\r\n",
      "Installing collected packages: openapi-schema-pydantic\r\n",
      "Successfully installed openapi-schema-pydantic-1.2.4\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/langchain-0.0.254-py3-none-any.whl\r\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (6.0)\r\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (2.0.17)\r\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (3.8.4)\r\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (4.0.2)\r\n",
      "Requirement already satisfied: dataclasses-json<0.6.0,>=0.5.7 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (0.5.9)\r\n",
      "Requirement already satisfied: langsmith<0.1.0,>=0.0.11 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (0.0.19)\r\n",
      "Requirement already satisfied: numexpr<3.0.0,>=2.8.4 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (2.8.4)\r\n",
      "Requirement already satisfied: numpy<2,>=1 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (1.23.5)\r\n",
      "Requirement already satisfied: openapi-schema-pydantic<2.0,>=1.2 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (1.2.4)\r\n",
      "Requirement already satisfied: pydantic<2,>=1 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (1.10.10)\r\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (2.31.0)\r\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /opt/conda/lib/python3.10/site-packages (from langchain==0.0.254) (8.2.2)\r\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.254) (23.1.0)\r\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.254) (3.1.0)\r\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.254) (6.0.4)\r\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.254) (1.9.2)\r\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.254) (1.3.3)\r\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain==0.0.254) (1.3.1)\r\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.3.0 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.254) (3.19.0)\r\n",
      "Requirement already satisfied: marshmallow-enum<2.0.0,>=1.5.1 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.254) (1.5.1)\r\n",
      "Requirement already satisfied: typing-inspect>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.254) (0.9.0)\r\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.10/site-packages (from pydantic<2,>=1->langchain==0.0.254) (4.6.3)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langchain==0.0.254) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langchain==0.0.254) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2->langchain==0.0.254) (2023.5.7)\r\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /opt/conda/lib/python3.10/site-packages (from SQLAlchemy<3,>=1.4->langchain==0.0.254) (2.0.2)\r\n",
      "Requirement already satisfied: packaging>=17.0 in /opt/conda/lib/python3.10/site-packages (from marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.254) (21.3)\r\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from typing-inspect>=0.4.0->dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.254) (1.0.0)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=17.0->marshmallow<4.0.0,>=3.3.0->dataclasses-json<0.6.0,>=0.5.7->langchain==0.0.254) (3.0.9)\r\n",
      "Installing collected packages: langchain\r\n",
      "Successfully installed langchain-0.0.254\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/peft-0.4.0-py3-none-any.whl\r\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (1.23.5)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (21.3)\r\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (5.9.3)\r\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (6.0)\r\n",
      "Requirement already satisfied: torch>=1.13.0 in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (2.0.0)\r\n",
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (4.30.2)\r\n",
      "Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (0.21.0)\r\n",
      "Requirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from peft==0.4.0) (0.3.1)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->peft==0.4.0) (3.0.9)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft==0.4.0) (3.12.2)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft==0.4.0) (4.6.3)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft==0.4.0) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft==0.4.0) (3.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->peft==0.4.0) (3.1.2)\r\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers->peft==0.4.0) (0.16.4)\r\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers->peft==0.4.0) (2023.6.3)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers->peft==0.4.0) (2.31.0)\r\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers->peft==0.4.0) (0.13.3)\r\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers->peft==0.4.0) (4.65.0)\r\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers->peft==0.4.0) (2023.6.0)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.13.0->peft==0.4.0) (2.1.3)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers->peft==0.4.0) (3.1.0)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers->peft==0.4.0) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers->peft==0.4.0) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers->peft==0.4.0) (2023.5.7)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.13.0->peft==0.4.0) (1.3.0)\r\n",
      "Installing collected packages: peft\r\n",
      "Successfully installed peft-0.4.0\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/transformers-4.31.0-py3-none-any.whl\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (3.12.2)\r\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (0.16.4)\r\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (1.23.5)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (21.3)\r\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (6.0)\r\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (2023.6.3)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (2.31.0)\r\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (0.13.3)\r\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (0.3.1)\r\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers==4.31.0) (4.65.0)\r\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31.0) (2023.6.0)\r\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.31.0) (4.6.3)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers==4.31.0) (3.0.9)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.31.0) (3.1.0)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.31.0) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.31.0) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers==4.31.0) (2023.5.7)\r\n",
      "Installing collected packages: transformers\r\n",
      "  Attempting uninstall: transformers\r\n",
      "    Found existing installation: transformers 4.30.2\r\n",
      "    Uninstalling transformers-4.30.2:\r\n",
      "      Successfully uninstalled transformers-4.30.2\r\n",
      "Successfully installed transformers-4.31.0\r\n",
      "Processing /kaggle/input/llm-package/LLM_Package/trl-0.4.7-py3-none-any.whl\r\n",
      "Requirement already satisfied: torch>=1.4.0 in /opt/conda/lib/python3.10/site-packages (from trl==0.4.7) (2.0.0)\r\n",
      "Requirement already satisfied: transformers>=4.18.0 in /opt/conda/lib/python3.10/site-packages (from trl==0.4.7) (4.31.0)\r\n",
      "Requirement already satisfied: numpy>=1.18.2 in /opt/conda/lib/python3.10/site-packages (from trl==0.4.7) (1.23.5)\r\n",
      "Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (from trl==0.4.7) (0.21.0)\r\n",
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from trl==0.4.7) (2.1.0)\r\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.4.0->trl==0.4.7) (3.12.2)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.4.0->trl==0.4.7) (4.6.3)\r\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.4.0->trl==0.4.7) (1.12)\r\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.4.0->trl==0.4.7) (3.1)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.4.0->trl==0.4.7) (3.1.2)\r\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (0.16.4)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (21.3)\r\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (6.0)\r\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (2023.6.3)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (2.31.0)\r\n",
      "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (0.13.3)\r\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (0.3.1)\r\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.18.0->trl==0.4.7) (4.65.0)\r\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate->trl==0.4.7) (5.9.3)\r\n",
      "Requirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (11.0.0)\r\n",
      "Requirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (0.3.6)\r\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (1.5.3)\r\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (3.2.0)\r\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (0.70.14)\r\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (2023.6.0)\r\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (3.8.4)\r\n",
      "Requirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets->trl==0.4.7) (0.18.0)\r\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (23.1.0)\r\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (3.1.0)\r\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (6.0.4)\r\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (4.0.2)\r\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (1.9.2)\r\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (1.3.3)\r\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->trl==0.4.7) (1.3.1)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->transformers>=4.18.0->trl==0.4.7) (3.0.9)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.18.0->trl==0.4.7) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.18.0->trl==0.4.7) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.18.0->trl==0.4.7) (2023.5.7)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.4.0->trl==0.4.7) (2.1.3)\r\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.4.7) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->trl==0.4.7) (2023.3)\r\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.4.0->trl==0.4.7) (1.3.0)\r\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.1->pandas->datasets->trl==0.4.7) (1.16.0)\r\n",
      "Installing collected packages: trl\r\n",
      "Successfully installed trl-0.4.7\r\n"
     ]
    }
   ],
   "source": [
    "!pip install /kaggle/input/llm-package/LLM_Package/accelerate-0.21.0-py3-none-any.whl \n",
    "!pip install /kaggle/input/llm-package/LLM_Package/bitsandbytes-0.41.1-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/einops-0.6.1-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/langsmith-0.0.19-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/openapi_schema_pydantic-1.2.4-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/langchain-0.0.254-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/peft-0.4.0-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/transformers-4.31.0-py3-none-any.whl\n",
    "!pip install /kaggle/input/llm-package/LLM_Package/trl-0.4.7-py3-none-any.whl\n",
    "# !pip install tiktoken\n",
    "# !pip install transformers_stream_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "562a56f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:35.749200Z",
     "iopub.status.busy": "2023-08-09T06:17:35.748831Z",
     "iopub.status.idle": "2023-08-09T06:17:51.460138Z",
     "shell.execute_reply": "2023-08-09T06:17:51.459131Z"
    },
    "papermill": {
     "duration": 15.727264,
     "end_time": "2023-08-09T06:17:51.463067",
     "exception": false,
     "start_time": "2023-08-09T06:17:35.735803",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.5\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:98: UserWarning: unable to load libtensorflow_io_plugins.so: unable to open file: libtensorflow_io_plugins.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io_plugins.so: undefined symbol: _ZN3tsl6StatusC1EN10tensorflow5error4CodeESt17basic_string_viewIcSt11char_traitsIcEENS_14SourceLocationE']\n",
      "  warnings.warn(f\"unable to load libtensorflow_io_plugins.so: {e}\")\n",
      "/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/__init__.py:104: UserWarning: file system plugins are not loaded: unable to open file: libtensorflow_io.so, from paths: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so']\n",
      "caused by: ['/opt/conda/lib/python3.10/site-packages/tensorflow_io/python/ops/libtensorflow_io.so: undefined symbol: _ZTVN10tensorflow13GcsFileSystemE']\n",
      "  warnings.warn(f\"file system plugins are not loaded: {e}\")\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import transformers\n",
    "\n",
    "from datasets import load_dataset, Dataset\n",
    "from typing import Optional, Union\n",
    "from dataclasses import dataclass\n",
    "from torch import nn \n",
    "from transformers import AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer, TrainingArguments, Trainer\n",
    "# from transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy\n",
    "from peft import LoraConfig, PeftModel, PeftConfig, get_peft_model\n",
    "from trl import SFTTrainer, DataCollatorForCompletionOnlyLM\n",
    "from langchain.prompts import PromptTemplate\n",
    "from IPython.display import Markdown, display\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn.preprocessing import normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea660a46",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:51.488479Z",
     "iopub.status.busy": "2023-08-09T06:17:51.488170Z",
     "iopub.status.idle": "2023-08-09T06:17:51.676161Z",
     "shell.execute_reply": "2023-08-09T06:17:51.675254Z"
    },
    "papermill": {
     "duration": 0.203139,
     "end_time": "2023-08-09T06:17:51.678447",
     "exception": false,
     "start_time": "2023-08-09T06:17:51.475308",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prompt</th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Which of the following statements accurately d...</td>\n",
       "      <td>MOND is a theory that reduces the observed mis...</td>\n",
       "      <td>MOND is a theory that increases the discrepanc...</td>\n",
       "      <td>MOND is a theory that explains the missing bar...</td>\n",
       "      <td>MOND is a theory that reduces the discrepancy ...</td>\n",
       "      <td>MOND is a theory that eliminates the observed ...</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Which of the following is an accurate definiti...</td>\n",
       "      <td>Dynamic scaling refers to the evolution of sel...</td>\n",
       "      <td>Dynamic scaling refers to the non-evolution of...</td>\n",
       "      <td>Dynamic scaling refers to the evolution of sel...</td>\n",
       "      <td>Dynamic scaling refers to the non-evolution of...</td>\n",
       "      <td>Dynamic scaling refers to the evolution of sel...</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>What is the significance of regularization in ...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Which of the following statements accurately d...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Which of the following statements accurately d...</td>\n",
       "      <td>The dimension of an object in a CW complex is ...</td>\n",
       "      <td>The dimension of an object in a CW complex is ...</td>\n",
       "      <td>The dimension of an object in a CW complex is ...</td>\n",
       "      <td>The dimension of an object in a CW complex is ...</td>\n",
       "      <td>The dimension of an object in a CW complex dep...</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                             prompt  \\\n",
       "0   0  Which of the following statements accurately d...   \n",
       "1   1  Which of the following is an accurate definiti...   \n",
       "2   2  What is the significance of regularization in ...   \n",
       "3   3  Which of the following statements accurately d...   \n",
       "4   4  Which of the following statements accurately d...   \n",
       "\n",
       "                                                   A  \\\n",
       "0  MOND is a theory that reduces the observed mis...   \n",
       "1  Dynamic scaling refers to the evolution of sel...   \n",
       "2  Regularizing the mass-energy of an electron wi...   \n",
       "3  The angular spacing of features in the diffrac...   \n",
       "4  The dimension of an object in a CW complex is ...   \n",
       "\n",
       "                                                   B  \\\n",
       "0  MOND is a theory that increases the discrepanc...   \n",
       "1  Dynamic scaling refers to the non-evolution of...   \n",
       "2  Regularizing the mass-energy of an electron wi...   \n",
       "3  The angular spacing of features in the diffrac...   \n",
       "4  The dimension of an object in a CW complex is ...   \n",
       "\n",
       "                                                   C  \\\n",
       "0  MOND is a theory that explains the missing bar...   \n",
       "1  Dynamic scaling refers to the evolution of sel...   \n",
       "2  Regularizing the mass-energy of an electron wi...   \n",
       "3  The angular spacing of features in the diffrac...   \n",
       "4  The dimension of an object in a CW complex is ...   \n",
       "\n",
       "                                                   D  \\\n",
       "0  MOND is a theory that reduces the discrepancy ...   \n",
       "1  Dynamic scaling refers to the non-evolution of...   \n",
       "2  Regularizing the mass-energy of an electron wi...   \n",
       "3  The angular spacing of features in the diffrac...   \n",
       "4  The dimension of an object in a CW complex is ...   \n",
       "\n",
       "                                                   E answer  \n",
       "0  MOND is a theory that eliminates the observed ...      D  \n",
       "1  Dynamic scaling refers to the evolution of sel...      A  \n",
       "2  Regularizing the mass-energy of an electron wi...      C  \n",
       "3  The angular spacing of features in the diffrac...      D  \n",
       "4  The dimension of an object in a CW complex dep...      A  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"/kaggle/input/llm-science-dataset/High Quality Dataset.csv\")\n",
    "data = data.dropna()\n",
    "\n",
    "def clean(text):\n",
    "    return text.strip()\n",
    "\n",
    "for column in data.columns:\n",
    "    if column != \"id\":\n",
    "        data[column] = data[column].astype(\"string\")\n",
    "        data[column] = data[column].apply(clean)\n",
    "\n",
    "# data = data.sample(len(data), random_state=42)\n",
    "# data[\"id\"] = range(len(data))\n",
    "# data.reset_index(drop=True, inplace=True)\n",
    "data = data[:500]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "23391836",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:51.703553Z",
     "iopub.status.busy": "2023-08-09T06:17:51.703270Z",
     "iopub.status.idle": "2023-08-09T06:17:51.929535Z",
     "shell.execute_reply": "2023-08-09T06:17:51.928406Z"
    },
    "papermill": {
     "duration": 0.242705,
     "end_time": "2023-08-09T06:17:51.933214",
     "exception": false,
     "start_time": "2023-08-09T06:17:51.690509",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading and preparing dataset csv/default to /root/.cache/huggingface/datasets/csv/default-a6e3f4d66142744f/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48863d8d2c7d4f098f3ff9026b257bfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bcc3f7b5ca24c88bf76ee8e5826cc44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset csv downloaded and prepared to /root/.cache/huggingface/datasets/csv/default-a6e3f4d66142744f/0.0.0/433e0ccc46f9880962cc2b12065189766fbb2bee57a221866138fb9203c83519. Subsequent calls will reuse this data.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/datasets/packaged_modules/csv/csv.py:154: FutureWarning: the 'mangle_dupe_cols' keyword is deprecated and will be removed in a future version. Please take steps to stop the use of 'mangle_dupe_cols'\n",
      "  csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.read_csv_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'prompt', 'A', 'B', 'C', 'D', 'E', 'answer'],\n",
       "    num_rows: 500\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.to_csv(\"./Shuffled Data.csv\", index=False)\n",
    "data = load_dataset(\"csv\", data_files=\"/kaggle/working/Shuffled Data.csv\", split=\"train\")\n",
    "# data = load_dataset(\"csv\", data_files=\"/kaggle/input/kaggle-llm-science-exam/train.csv\", split=\"train\")\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4581c956",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:51.959436Z",
     "iopub.status.busy": "2023-08-09T06:17:51.959125Z",
     "iopub.status.idle": "2023-08-09T06:17:51.964647Z",
     "shell.execute_reply": "2023-08-09T06:17:51.963641Z"
    },
    "papermill": {
     "duration": 0.020726,
     "end_time": "2023-08-09T06:17:51.966730",
     "exception": false,
     "start_time": "2023-08-09T06:17:51.946004",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "template = \"\"\"Answer the following multiple choice question by giving the most appropriate response. Answer should be one among [A, B, C, D, E]\\n\n",
    "Question: {prompt}\\n\n",
    "A) {a}\\n\n",
    "B) {b}\\n\n",
    "C) {c}\\n\n",
    "D) {d}\\n\n",
    "E) {e}\\n\n",
    "### Answer: {answer}\"\"\"\n",
    "\n",
    "prompt = PromptTemplate(template=template, input_variables=[\"prompt\", \"a\", \"b\", \"c\", \"d\", \"e\", \"answer\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "47371be2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:51.993130Z",
     "iopub.status.busy": "2023-08-09T06:17:51.992363Z",
     "iopub.status.idle": "2023-08-09T06:17:52.000106Z",
     "shell.execute_reply": "2023-08-09T06:17:51.999082Z"
    },
    "papermill": {
     "duration": 0.023284,
     "end_time": "2023-08-09T06:17:52.002251",
     "exception": false,
     "start_time": "2023-08-09T06:17:51.978967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "Answer the following multiple choice question by giving the most appropriate response. Answer should be one among [A, B, C, D, E]\n",
       "\n",
       "Question: Which of the following statements accurately describes the impact of Modified Newtonian Dynamics (MOND) on the observed \"missing baryonic mass\" discrepancy in galaxy clusters?\n",
       "\n",
       "A) MOND is a theory that reduces the observed missing baryonic mass in galaxy clusters by postulating the existence of a new form of matter called \"fuzzy dark matter.\"\n",
       "\n",
       "B) MOND is a theory that increases the discrepancy between the observed missing baryonic mass in galaxy clusters and the measured velocity dispersions from a factor of around 10 to a factor of about 20.\n",
       "\n",
       "C) MOND is a theory that explains the missing baryonic mass in galaxy clusters that was previously considered dark matter by demonstrating that the mass is in the form of neutrinos and axions.\n",
       "\n",
       "D) MOND is a theory that reduces the discrepancy between the observed missing baryonic mass in galaxy clusters and the measured velocity dispersions from a factor of around 10 to a factor of about 2.\n",
       "\n",
       "E) MOND is a theory that eliminates the observed missing baryonic mass in galaxy clusters by imposing a new mathematical formulation of gravity that does not require the existence of dark matter.\n",
       "\n",
       "### Answer: D"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample = data[0]\n",
    "display(Markdown(prompt.format(prompt=sample[\"prompt\"], a=sample[\"A\"], \n",
    "                               b=sample[\"B\"], c=sample[\"C\"], d=sample[\"D\"], \n",
    "                               e=sample[\"E\"], answer=sample[\"answer\"])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fd3cc0e9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:52.029882Z",
     "iopub.status.busy": "2023-08-09T06:17:52.029603Z",
     "iopub.status.idle": "2023-08-09T06:17:52.184112Z",
     "shell.execute_reply": "2023-08-09T06:17:52.183076Z"
    },
    "papermill": {
     "duration": 0.171026,
     "end_time": "2023-08-09T06:17:52.186645",
     "exception": false,
     "start_time": "2023-08-09T06:17:52.015619",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a0182849d374b24ace92a3eebc8c28a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['id', 'prompt', 'A', 'B', 'C', 'D', 'E', 'answer', 'text'],\n",
       "    num_rows: 500\n",
       "})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_text(example):\n",
    "    text = prompt.format(prompt=example[\"prompt\"], a=example[\"A\"], \n",
    "                         b=example[\"B\"], c=example[\"C\"], d=example[\"D\"], \n",
    "                         e=example[\"E\"], answer=example[\"answer\"])\n",
    "    return {\"text\": text}\n",
    "\n",
    "data = data.map(format_text)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a93d0f90",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:52.213676Z",
     "iopub.status.busy": "2023-08-09T06:17:52.213408Z",
     "iopub.status.idle": "2023-08-09T06:17:52.668128Z",
     "shell.execute_reply": "2023-08-09T06:17:52.667099Z"
    },
    "papermill": {
     "duration": 0.470911,
     "end_time": "2023-08-09T06:17:52.670302",
     "exception": false,
     "start_time": "2023-08-09T06:17:52.199391",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d851f15680ab4f5482cfcb6e7e762d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAHFCAYAAAAHcXhbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA+YUlEQVR4nO3dd3RUdf7/8ddAeiH0hNBC70WKCIuGgHQLYkGasOoKUiRiARYXgiv1q4iKgLor4CpiA0RBJVRFYGlGihUNRSFEEUkoCSXv3x+czI8hCSQhMLnZ5+OcnMO99zP3vt8zk8mL28ZlZiYAAACHKubtAgAAAK4EYQYAADgaYQYAADgaYQYAADgaYQYAADgaYQYAADgaYQYAADgaYQYAADgaYQYAADgaYQYFat68eXK5XO6fgIAARUREKCYmRpMnT1ZycnKWx8TFxcnlcuVpOydPnlRcXJzWrl2bp8dlt62oqCjdcssteVrP5SxYsEAzZszIdpnL5VJcXFyBbq+grVq1Si1atFBwcLBcLpeWLFmSZUy7du08Xuucfgqy10mTJmVbS06OHDmiMWPGqH79+goODlZYWJjq1q2r/v37a8eOHQVW1/+idu3aqWHDht4uI0fLly/P8b3ncrk0bNiwa1sQriofbxeAomnu3LmqW7euzpw5o+TkZK1fv15Tp07Vs88+q3feeUc333yze+yDDz6oLl265Gn9J0+e1IQJEySd/1DNrfxsKz8WLFigXbt2KTY2NsuyjRs3qlKlSle9hvwyM91zzz2qXbu2li5dquDgYNWpUyfLuFmzZiklJcU9vWzZMj3zzDPu1z5TQfY6adIk3XXXXerRo8dlxx4/flw33HCDjh8/rieeeEJNmjTRqVOn9MMPP2jRokVKSEhQ48aNC6w2FC7Lly/Xyy+/XOj/44CCQZjBVdGwYUO1aNHCPX3nnXfq0UcfVdu2bdWzZ0/9+OOPCg8Pl3T+j93V/uN+8uRJBQUFXZNtXc4NN9zg1e1fzsGDB/XHH3/ojjvuUIcOHXIcV79+fY/p7777TlLW195b3nvvPe3Zs0erV69WTEyMx7KRI0cqIyPDS5UBKGgcZsI1U6VKFT333HNKTU3VK6+84p6f3aGf1atXq127dipTpowCAwNVpUoV3XnnnTp58qT27t2rcuXKSZImTJjgPpwxcOBAj/Vt375dd911l0qVKqUaNWrkuK1MixcvVuPGjRUQEKDq1avrxRdf9FieeQht7969HvPXrl0rl8vlPuTVrl07LVu2TPv27fM43JIpu0Mvu3bt0u23365SpUopICBATZs21fz587Pdzttvv62xY8cqMjJSJUqU0M0336zvv/8+5yf+AuvXr1eHDh0UGhqqoKAgtWnTRsuWLXMvj4uLc4e9UaNGyeVyKSoqKlfrzsk777yj1q1bKzg4WCEhIercubO++uorj5p8fX31+OOPezwu8/n+97//Len883bixAnNnz/f/Zxeaq/ckSNHJEkVKlTIdnmxYp4ffz/++KP69Omj8uXLy9/fX/Xq1dPLL7+c5XHfffedunTpoqCgIJUtW1aDBw/WRx995PEekM4fvsx8T16oXbt2WepOSUnR448/rmrVqsnPz08VK1ZUbGysTpw44TEu8/DIf/7zH9WrV09BQUFq0qSJPv7442zr7N27t8LDw+Xv768qVarovvvuU3p6untMUlKSBg0apEqVKsnPz0/VqlXThAkTdPbs2Wyfs/y43OsvSQMHDlRISIj27Nmjbt26KSQkRJUrV9Zjjz3mUa8k/fLLL7rrrrsUGhqqkiVLqm/fvtqyZYtcLpfmzZvnXl/ma3fh7+DFv7uXex5/++03PfTQQ6pcubL8/f1Vrlw5/eUvf9HKlSsL7PlBATGgAM2dO9ck2ZYtW7Jdfvz4cStevLh16NDBPW/8+PF24VsxMTHRAgICrGPHjrZkyRJbu3atvfXWW9a/f387evSopaWl2aeffmqS7IEHHrCNGzfaxo0bbc+ePR7rq1q1qo0aNcri4+NtyZIl2W7LzKxq1apWsWJFq1Klir3++uu2fPly69u3r0my//u//8vSW2Jiosfj16xZY5JszZo1Zma2e/du+8tf/mIRERHu2jZu3OgeL8nGjx/vnv7uu+8sNDTUatSoYW+88YYtW7bMevfubZJs6tSpWbYTFRVlffv2tWXLltnbb79tVapUsVq1atnZs2cv+dqsXbvWfH19rXnz5vbOO+/YkiVLrFOnTuZyuWzhwoVmZnbgwAFbtGiRSbLhw4fbxo0bbfv27Zdc78XPz4Wv/cSJE83lctn9999vH3/8sS1atMhat25twcHBtnv3bve4KVOmmCT78MMPzcxs165dFhQUZP369XOP2bhxowUGBlq3bt3cz+mF67jY+vXrTZK1bNnSFi9ebL///nuOY3fv3m1hYWHWqFEje+ONN2zFihX22GOPWbFixSwuLs49LikpycqXL28VK1a0uXPnut8rVapU8XgPmJ1/Xw0YMCDLtqKjoy06Oto9feLECWvatKmVLVvWpk+fbitXrrQXXnjBwsLCrH379paRkeEem/n6X3/99fbuu+/a8uXLrV27dubj42M//fSTe1xCQoKFhIRYVFSUzZkzx1atWmVvvvmm3XPPPZaSkmJmZocOHbLKlStb1apV7ZVXXrGVK1faP//5T/P397eBAwfm+Fxd2EeDBg0uOSa3r/+AAQPMz8/P6tWrZ88++6ytXLnSxo0bZy6XyyZMmOAed/z4catZs6aVLl3aXn75Zfvss8/s0UcftWrVqpkkmzt3rpmZ7dmzx+666y6T5PE7mJaWlqfnsXPnzlauXDl79dVXbe3atbZkyRIbN26c+/cFhQdhBgXqcmHGzCw8PNzq1avnnr44YLz//vsmyRISEnJcx2+//ZYlFFy8vnHjxuW47EJVq1Y1l8uVZXsdO3a0EiVK2IkTJzx6u1yYMTPr3r27Va1aNdvaL6773nvvNX9/f9u/f7/HuK5du1pQUJD9+eefHtvp1q2bx7h3333X/aF9KTfccIOVL1/eUlNT3fPOnj1rDRs2tEqVKrn/aCYmJmYJcrlx8Wu/f/9+8/HxseHDh3uMS01NtYiICLvnnnvc8zIyMqxbt25WsmRJ27Vrl9WvX9/q1q1rx48f93hscHBwtgEhJ08//bT5+fmZJJNk1apVs8GDB9vXX3/tMa5z585WqVIlO3bsmMf8YcOGWUBAgP3xxx9mZjZq1Kgc3yv5DTOTJ0+2YsWKZfmdyfw9WL58uXueJAsPD3cHErPzAatYsWI2efJk97z27dtbyZIlLTk5OcfnZtCgQRYSEmL79u3zmP/ss8+apEsGxcw+LhVm8vL6DxgwwCTZu+++6zG2W7duVqdOHff0yy+/bJLsk08+ydLLhWHGzGzo0KFZftcz5fZ5DAkJsdjY2Bx7ROHBYSZcc2Z2yeVNmzaVn5+fHnroIc2fP18///xzvrZz55135npsgwYN1KRJE495ffr0UUpKirZv356v7efW6tWr1aFDB1WuXNlj/sCBA3Xy5Elt3LjRY/5tt93mMZ15Euu+ffty3MaJEyf03//+V3fddZdCQkLc84sXL67+/fvrl19+yfWhqtz67LPPdPbsWd133306e/as+ycgIEDR0dEeh2RcLpfeeOMNhYaGqkWLFkpMTNS7776r4ODgK6rhH//4h/bv36/XX39dgwYNUkhIiObMmaPmzZvr7bffliSlpaVp1apVuuOOOxQUFORRa7du3ZSWlqZNmzZJktasWZPjeyW/Pv74YzVs2FBNmzb12Hbnzp2zHLqSpJiYGIWGhrqnw8PDVb58effrf/LkSa1bt0733HOP+3BsTtuNiYlRZGSkx3a7du0qSVq3bl2+e5Ly9vpL598Dt956q8e8xo0be7yv161bp9DQ0Cwn8ffu3TvP9V3ueZSk66+/XvPmzdMzzzyjTZs26cyZM3neDq4NwgyuqRMnTujIkSOKjIzMcUyNGjW0cuVKlS9fXkOHDlWNGjVUo0YNvfDCC3naVk7nSmQnIiIix3mZ515cLUeOHMm21szn6OLtlylTxmPa399fknTq1Kkct3H06FGZWZ62c6UOHz4sSWrZsqV8fX09ft555x39/vvvHuPLlCmj2267TWlpaerSpYsaNWpUIHWEh4frr3/9q+bMmaMdO3Zo3bp18vPz04gRIySd7/vs2bN66aWXstTZrVs3SXLXeuTIkUu+V/Lj8OHD2rFjR5Zth4aGysyyfZ4u5u/v7379jx49qnPnzl32RPfDhw/ro48+yrLdBg0aSFKW7eanLyn3r39QUJACAgKy9JWWluaePnLkiPvCgQtlN+9yLvc8SufP9xkwYID+9a9/qXXr1ipdurTuu+8+JSUl5Xl7uLq4mgnX1LJly3Tu3LnLXk5944036sYbb9S5c+e0detWvfTSS4qNjVV4eLjuvffeXG0rL/euye7DKXNe5ode5gftxSckXumHfpkyZXTo0KEs8w8ePChJKlu27BWtX5JKlSqlYsWKXfXtXChzfe+//76qVq162fHx8fGaPXu2rr/+ei1evFgffPBBnvau5dZNN92kTp06acmSJUpOTlapUqXce6iGDh2a7WOqVasm6fxrdan3yoUCAgKyvFek8++XC5/rsmXLKjAwUK+//nq2287r61K6dGkVL15cv/zyyyXHlS1bVo0bN9bEiROzXX6p/3DkRl5f/9woU6aMNm/enGX+1QoXZcuW1YwZMzRjxgzt379fS5cu1ejRo5WcnKxPP/30qmwT+UOYwTWzf/9+Pf744woLC9OgQYNy9ZjixYurVatWqlu3rt566y1t375d9957b672RuTF7t279fXXX3scPliwYIFCQ0PVrFkzSXJf1bNjxw6P+64sXbo0y/ou/h/epXTo0EGLFy/WwYMHPf6AvPHGGwoKCiqQS7mDg4PVqlUrLVq0SM8++6wCAwMlSRkZGXrzzTdVqVIl1a5d+4q3c6HOnTvLx8dHP/3002VDyaFDh9SvXz9FR0crPj5ePXv21AMPPKBmzZq5g4SUt+f18OHDKleuXJarls6dO6cff/xRQUFBKlmypPz8/BQTE6OvvvpKjRs3lp+fX47rjImJ0bRp07J9r1wsKioqy435fvjhB33//fceAeWWW27RpEmTVKZMGY9e8yswMFDR0dF67733NHHixBzD0C233KLly5erRo0aKlWq1BVv92J5ef1zKzo6Wu+++64++eQT9+EwSVq4cGGWsRd+RmS+369ElSpVNGzYMK1atUpffvnlFa8PBYswg6ti165d7mPkycnJ+uKLLzR37lwVL15cixcvvuSx/Dlz5mj16tXq3r27qlSporS0NPf/WjNvthcaGqqqVavqww8/VIcOHVS6dGmVLVs235cRR0ZG6rbbblNcXJwqVKigN998U/Hx8Zo6daqCgoIknd9dXqdOHT3++OM6e/asSpUqpcWLF2v9+vVZ1teoUSMtWrRIs2fPVvPmzVWsWLEc770yfvx49/kL48aNU+nSpfXWW29p2bJlmjZtmsLCwvLV08UmT56sjh07KiYmRo8//rj8/Pw0a9Ys7dq1S2+//Xae78J8OVFRUXr66ac1duxY/fzzz+rSpYtKlSqlw4cPa/PmzQoODtaECRN07tw59e7dWy6XSwsWLFDx4sU1b948NW3aVL169dL69evdAaNRo0Zau3atPvroI1WoUEGhoaHZ3tBPOn/Z7SuvvKI+ffqoZcuWCgsL0y+//KJ//etf2r17t8aNG+de7wsvvKC2bdvqxhtv1MMPP6yoqCilpqZqz549+uijj7R69WpJUmxsrF5//XV1795dzzzzjMLDw/XWW2+577Fzof79+6tfv34aMmSI7rzzTu3bt0/Tpk3L8t6PjY3VBx98oJtuukmPPvqoGjdurIyMDO3fv18rVqzQY489platWuXpuZ8+fbratm2rVq1aafTo0apZs6YOHz6spUuX6pVXXlFoaKiefvppxcfHq02bNnrkkUdUp04dpaWlae/evVq+fLnmzJlz2UNVKSkpev/997PML1eunKKjo3P1+ufFgAED9Pzzz6tfv3565plnVLNmTX3yySf67LPPJHlebp95mHLq1Knq2rWrihcvftmweqFjx44pJiZGffr0Ud26dRUaGqotW7bo008/Vc+ePfNUN64BL5+AjCIm84qWzB8/Pz8rX768RUdH26RJk7K9uuLiK4w2btxod9xxh1WtWtX8/f2tTJkyFh0dbUuXLvV43MqVK+26664zf39/k+S+ciRzfb/99ttlt2V2/qqT7t272/vvv28NGjQwPz8/i4qKsunTp2d5/A8//GCdOnWyEiVKWLly5Wz48OG2bNmyLFey/PHHH3bXXXdZyZIlzeVyeWxT2VyFtXPnTrv11lstLCzM/Pz8rEmTJh5XZpj9/6uZ3nvvPY/5mVcfXTw+O1988YW1b9/egoODLTAw0G644Qb76KOPsl3flV7NlGnJkiUWExNjJUqUMH9/f6tatardddddtnLlSjMzGzt2rBUrVsxWrVrl8bgNGzaYj4+PjRgxwj0vISHB/vKXv1hQUJBJ8rgq6GLffPONPfbYY9aiRQsrV66c+fj4WKlSpSw6Otr+85//ZBmfmJho999/v1WsWNF8fX2tXLly1qZNG3vmmWeyrLdjx44WEBBgpUuXtgceeMA+/PDDLO+BjIwMmzZtmlWvXt0CAgKsRYsWtnr16ixXM5mdv+T4qaeesjp16pifn5/7MvFHH33UkpKS3OMk2dChQ7PUnt2VU998843dfffdVqZMGfPz87MqVarYwIED3Zcnm52/KvCRRx6xatWqma+vr5UuXdqaN29uY8eOzXIl2cWio6M9ftcv/Lmwv8u9/mbnr2YKDg7Oso3sfl/3799vPXv2tJCQEAsNDbU777zTli9f7nFpv5lZenq6Pfjgg1auXDn372DmlYi5eR7T0tJs8ODB1rhxYytRooQFBgZanTp1bPz48e4rHFF4uMwuc2kJAOCS1q5dq5iYGK1ZsyZPX6+BgjFp0iQ99dRT2r9/v9fv8A3v4DATAMAxZs6cKUnu735bvXq1XnzxRfXr148g8z+MMAMAcIygoCA9//zz2rt3r9LT01WlShWNGjVKTz31lLdLgxdxmAkAADgaN80DAACORpgBAACORpgBAACOVuRPAM7IyNDBgwcVGhpa4DcFAwAAV4eZKTU1VZGRkVnu5H2xIh9mDh48mOXbiAEAgDMcOHDgspfdF/kwk/kV7wcOHFCJEiW8XA0AAMiNlJQUVa5c2f13/FKKfJjJPLRUokQJwgwAAA6Tm1NEOAEYAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4mo+3C/hfFjV6Wb4fu3dK9wKsBAAA52LPDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDQfbxeA/IkavSzfj907pXsBVgIAgHexZwYAADgaYQYAADgaYQYAADhaoQkzkydPlsvlUmxsrHuemSkuLk6RkZEKDAxUu3bttHv3bu8VCQAACp1CEWa2bNmiV199VY0bN/aYP23aNE2fPl0zZ87Uli1bFBERoY4dOyo1NdVLlQIAgMLG62Hm+PHj6tu3r1577TWVKlXKPd/MNGPGDI0dO1Y9e/ZUw4YNNX/+fJ08eVILFizwYsUAAKAw8XqYGTp0qLp3766bb77ZY35iYqKSkpLUqVMn9zx/f39FR0drw4YNOa4vPT1dKSkpHj8AAKDo8up9ZhYuXKjt27dry5YtWZYlJSVJksLDwz3mh4eHa9++fTmuc/LkyZowYULBFgoAAAotr+2ZOXDggEaMGKE333xTAQEBOY5zuVwe02aWZd6FxowZo2PHjrl/Dhw4UGA1AwCAwsdre2a2bdum5ORkNW/e3D3v3Llz+vzzzzVz5kx9//33ks7voalQoYJ7THJycpa9NRfy9/eXv7//1SscAAAUKl7bM9OhQwft3LlTCQkJ7p8WLVqob9++SkhIUPXq1RUREaH4+Hj3Y06fPq1169apTZs23iobAAAUMl7bMxMaGqqGDRt6zAsODlaZMmXc82NjYzVp0iTVqlVLtWrV0qRJkxQUFKQ+ffp4o2QAAFAIFeovmnzyySd16tQpDRkyREePHlWrVq20YsUKhYaGers0AABQSLjMzLxdxNWUkpKisLAwHTt2TCVKlPB2OR6u5JuvrwTfmg0AKOzy8vfb6/eZAQAAuBKF+jAT4GRXsueNvWcAkHvsmQEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI5GmAEAAI7m4+0CgMIsavQyb5cAALgM9swAAABHI8wAAABHI8wAAABHI8wAAABHI8wAAABHI8wAAABHI8wAAABH4z4zuGau5J4te6d0L8BKAABFCXtmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAoxFmAACAo3k1zMyePVuNGzdWiRIlVKJECbVu3VqffPKJe7mZKS4uTpGRkQoMDFS7du20e/duL1YMAAAKG6+GmUqVKmnKlCnaunWrtm7dqvbt2+v22293B5Zp06Zp+vTpmjlzprZs2aKIiAh17NhRqamp3iwbAAAUIl4NM7feequ6deum2rVrq3bt2po4caJCQkK0adMmmZlmzJihsWPHqmfPnmrYsKHmz5+vkydPasGCBd4sGwAAFCKF5pyZc+fOaeHChTpx4oRat26txMREJSUlqVOnTu4x/v7+io6O1oYNG3JcT3p6ulJSUjx+AABA0eXj7QJ27typ1q1bKy0tTSEhIVq8eLHq16/vDizh4eEe48PDw7Vv374c1zd58mRNmDDhqtZ8oajRy67ZtgAAQFZe3zNTp04dJSQkaNOmTXr44Yc1YMAAffPNN+7lLpfLY7yZZZl3oTFjxujYsWPunwMHDly12gEAgPd5fc+Mn5+fatasKUlq0aKFtmzZohdeeEGjRo2SJCUlJalChQru8cnJyVn21lzI399f/v7+V7doAABQaHh9z8zFzEzp6emqVq2aIiIiFB8f7152+vRprVu3Tm3atPFihQAAoDDx6p6Zv//97+ratasqV66s1NRULVy4UGvXrtWnn34ql8ul2NhYTZo0SbVq1VKtWrU0adIkBQUFqU+fPt4sGwAAFCJeDTOHDx9W//79dejQIYWFhalx48b69NNP1bFjR0nSk08+qVOnTmnIkCE6evSoWrVqpRUrVig0NNSbZQMAgELEq2Hm3//+9yWXu1wuxcXFKS4u7toUBAAAHKfQnTMDAACQF16/mgnIjSu5n8/eKd0LsBIAQGHDnhkAAOBohBkAAOBohBkAAOBohBkAAOBohBkAAOBohBkAAOBohBkAAOBohBkAAOBo3DQPKIS4SSAA5B57ZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKNxnxnkyZXc/wQAgKuBPTMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDRCDMAAMDR8hVmqlevriNHjmSZ/+eff6p69epXXBQAAEBu5SvM7N27V+fOncsyPz09Xb/++usVFwUAAJBbPnkZvHTpUve/P/vsM4WFhbmnz507p1WrVikqKqrAigMAALicPIWZHj16SJJcLpcGDBjgsczX11dRUVF67rnnCqw4AACAy8lTmMnIyJAkVatWTVu2bFHZsmWvSlEAAAC5lacwkykxMbGg6wAAAMiXfIUZSVq1apVWrVql5ORk9x6bTK+//voVFwYUlKjRy7xdAgDgKspXmJkwYYKefvpptWjRQhUqVJDL5SrougAAAHIlX2Fmzpw5mjdvnvr371/Q9QAAAORJvu4zc/r0abVp06agawEAAMizfIWZBx98UAsWLCjoWgAAAPIsX4eZ0tLS9Oqrr2rlypVq3LixfH19PZZPnz69QIoDAAC4nHyFmR07dqhp06aSpF27dnks42RgAABwLeUrzKxZs6ag6wAAAMiXfN9nBkDhdCX31dk7pXsBVgIA10a+wkxMTMwlDyetXr063wUBAADkRb7CTOb5MpnOnDmjhIQE7dq1K8sXUAIAAFxN+Qozzz//fLbz4+LidPz48SsqCAAAIC/ydZ+ZnPTr14/vZQIAANdUgYaZjRs3KiAgoCBXCQAAcEn5OszUs2dPj2kz06FDh7R161b94x//KJDCAAAAciNfYSYsLMxjulixYqpTp46efvppderUqUAKAwAAyI18hZm5c+cWdB0AAAD5ckU3zdu2bZu+/fZbuVwu1a9fX9ddd11B1QUAAJAr+QozycnJuvfee7V27VqVLFlSZqZjx44pJiZGCxcuVLly5Qq6TgAAgGzl62qm4cOHKyUlRbt379Yff/yho0ePateuXUpJSdEjjzxS0DUCAADkKF97Zj799FOtXLlS9erVc8+rX7++Xn75ZU4ABgAA11S+9sxkZGTI19c3y3xfX19lZGRccVEAAAC5la8w0759e40YMUIHDx50z/v111/16KOPqkOHDgVWHAAAwOXkK8zMnDlTqampioqKUo0aNVSzZk1Vq1ZNqampeumllwq6RgAAgBzl65yZypUra/v27YqPj9d3330nM1P9+vV18803F3R9AAAAl5SnPTOrV69W/fr1lZKSIknq2LGjhg8frkceeUQtW7ZUgwYN9MUXX1yVQgEAALKTpzAzY8YM/e1vf1OJEiWyLAsLC9OgQYM0ffr0AisOAADgcvIUZr7++mt16dIlx+WdOnXStm3brrgoAACA3MpTmDl8+HC2l2Rn8vHx0W+//XbFRQEAAORWnsJMxYoVtXPnzhyX79ixQxUqVMj1+iZPnqyWLVsqNDRU5cuXV48ePfT99997jDEzxcXFKTIyUoGBgWrXrp12796dl7IBAEARlqcw061bN40bN05paWlZlp06dUrjx4/XLbfckuv1rVu3TkOHDtWmTZsUHx+vs2fPqlOnTjpx4oR7zLRp0zR9+nTNnDlTW7ZsUUREhDp27KjU1NS8lA4AAIool5lZbgcfPnxYzZo1U/HixTVs2DDVqVNHLpdL3377rV5++WWdO3dO27dvV3h4eL6K+e2331S+fHmtW7dON910k8xMkZGRio2N1ahRoyRJ6enpCg8P19SpUzVo0KDLrjMlJUVhYWE6duxYticuX6mo0csKfJ1X294p3fP9WCf2i9y7kvcGABSkvPz9ztN9ZsLDw7VhwwY9/PDDGjNmjDJzkMvlUufOnTVr1qx8BxlJOnbsmCSpdOnSkqTExEQlJSV5fN+Tv7+/oqOjtWHDhmzDTHp6utLT093TmZeRAwCAoinPN82rWrWqli9frqNHj2rPnj0yM9WqVUulSpW6okLMTCNHjlTbtm3VsGFDSVJSUpIkZQlI4eHh2rdvX7brmTx5siZMmHBFtQAAAOfI1x2AJalUqVJq2bJlgRUybNgw7dixQ+vXr8+yzOVyeUybWZZ5mcaMGaORI0e6p1NSUlS5cuUCqxMAABQu+Q4zBWn48OFaunSpPv/8c1WqVMk9PyIiQtL5PTQXXiWVnJyc4+Esf39/+fv7X92CAQBAoZGvL5osKGamYcOGadGiRVq9erWqVavmsbxatWqKiIhQfHy8e97p06e1bt06tWnT5lqXCwAACiGv7pkZOnSoFixYoA8//FChoaHuc2TCwsIUGBgol8ul2NhYTZo0SbVq1VKtWrU0adIkBQUFqU+fPt4sHQAAFBJeDTOzZ8+WJLVr185j/ty5czVw4EBJ0pNPPqlTp05pyJAhOnr0qFq1aqUVK1YoNDT0GlcLAAAKI6+Gmdzc4sblcikuLk5xcXFXvyAAAOA4Xj1nBgAA4EoRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMVim/NBuB8UaOX5fuxe6d0L8BKAPyvYc8MAABwNMIMAABwNMIMAABwNMIMAABwNMIMAABwNMIMAABwNMIMAABwNMIMAABwNG6a9z/oSm5uBlwN3HAPwJVgzwwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0wgwAAHA0H28XAKDwiBq9zNslAECesWcGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4GmEGAAA4mlfDzOeff65bb71VkZGRcrlcWrJkicdyM1NcXJwiIyMVGBiodu3aaffu3d4pFgAAFEpeDTMnTpxQkyZNNHPmzGyXT5s2TdOnT9fMmTO1ZcsWRUREqGPHjkpNTb3GlQIAgMLKx5sb79q1q7p27ZrtMjPTjBkzNHbsWPXs2VOSNH/+fIWHh2vBggUaNGjQtSwVAAAUUoX2nJnExEQlJSWpU6dO7nn+/v6Kjo7Whg0bcnxcenq6UlJSPH4AAEDR5dU9M5eSlJQkSQoPD/eYHx4ern379uX4uMmTJ2vChAlXtTYA8Jao0cvy/di9U7oXYCW558Sa4SyFds9MJpfL5TFtZlnmXWjMmDE6duyY++fAgQNXu0QAAOBFhXbPTEREhKTze2gqVKjgnp+cnJxlb82F/P395e/vf9XrAwAAhUOh3TNTrVo1RUREKD4+3j3v9OnTWrdundq0aePFygAAQGHi1T0zx48f1549e9zTiYmJSkhIUOnSpVWlShXFxsZq0qRJqlWrlmrVqqVJkyYpKChIffr08WLVAACgMPFqmNm6datiYmLc0yNHjpQkDRgwQPPmzdOTTz6pU6dOaciQITp69KhatWqlFStWKDQ01FslAwCAQsarYaZdu3YysxyXu1wuxcXFKS4u7toVBQAAHKXQnjMDAACQG4X2aiYAuNqu5P4nV4J7pwAFiz0zAADA0QgzAADA0QgzAADA0QgzAADA0QgzAADA0QgzAADA0QgzAADA0bjPDADgsrx1Tx4gN9gzAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHI0wAwAAHM3H2wUAwJWIGr3M2yXkmRNrBgoz9swAAABHI8wAAABHI8wAAABHI8wAAABHI8wAAABHI8wAAABHI8wAAABH4z4zAPA/won3t3FizXundPd2Cf9z2DMDAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAcjTADAAAczcfbBQAAUJREjV6W78fundK9ACvJPSfWfCH2zAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEcjzAAAAEfz8XYBAADgvKjRy/L92L1TuhdgJc7CnhkAAOBohBkAAOBojggzs2bNUrVq1RQQEKDmzZvriy++8HZJAACgkCj0Yeadd95RbGysxo4dq6+++ko33nijunbtqv3793u7NAAAUAgU+jAzffp0PfDAA3rwwQdVr149zZgxQ5UrV9bs2bO9XRoAACgECnWYOX36tLZt26ZOnTp5zO/UqZM2bNjgpaoAAEBhUqgvzf7999917tw5hYeHe8wPDw9XUlJSto9JT09Xenq6e/rYsWOSpJSUlKtSY0b6yauyXgAA8uJK/s5dyd+yq/X3NXO9ZnbZsYU6zGRyuVwe02aWZV6myZMna8KECVnmV65c+arUBgBAYRA2o2huNzU1VWFhYZccU6jDTNmyZVW8ePEse2GSk5Oz7K3JNGbMGI0cOdI9nZGRoT/++ENlypTJMQClpKSocuXKOnDggEqUKFFwDRQS9OdsRb0/qej3SH/ORn/eYWZKTU1VZGTkZccW6jDj5+en5s2bKz4+XnfccYd7fnx8vG6//fZsH+Pv7y9/f3+PeSVLlszV9kqUKFGoXsiCRn/OVtT7k4p+j/TnbPR37V1uj0ymQh1mJGnkyJHq37+/WrRoodatW+vVV1/V/v37NXjwYG+XBgAACoFCH2Z69eqlI0eO6Omnn9ahQ4fUsGFDLV++XFWrVvV2aQAAoBAo9GFGkoYMGaIhQ4ZctfX7+/tr/PjxWQ5PFRX052xFvT+p6PdIf85Gf4Wfy3JzzRMAAEAhVahvmgcAAHA5hBkAAOBohBkAAOBohBkAAOBoRTLMTJ48WS1btlRoaKjKly+vHj166Pvvv/cYY2aKi4tTZGSkAgMD1a5dO+3evdtjTHp6uoYPH66yZcsqODhYt912m3755Zdr2UquTJ48WS6XS7Gxse55RaG/X3/9Vf369VOZMmUUFBSkpk2batu2be7lTu7x7Nmzeuqpp1StWjUFBgaqevXqevrpp5WRkeEe46T+Pv/8c916662KjIyUy+XSkiVLPJYXVC9Hjx5V//79FRYWprCwMPXv319//vnnVe7uvEv1eObMGY0aNUqNGjVScHCwIiMjdd999+ngwYMe6yjMPV7uNbzQoEGD5HK5NGPGDI/5Tu/v22+/1W233aawsDCFhobqhhtu0P79+93Lndzf8ePHNWzYMFWqVEmBgYGqV6+eZs+e7TGmMPd3WVYEde7c2ebOnWu7du2yhIQE6969u1WpUsWOHz/uHjNlyhQLDQ21Dz74wHbu3Gm9evWyChUqWEpKinvM4MGDrWLFihYfH2/bt2+3mJgYa9KkiZ09e9YbbWVr8+bNFhUVZY0bN7YRI0a45zu9vz/++MOqVq1qAwcOtP/+97+WmJhoK1eutD179rjHOLnHZ555xsqUKWMff/yxJSYm2nvvvWchISE2Y8YM9xgn9bd8+XIbO3asffDBBybJFi9e7LG8oHrp0qWLNWzY0DZs2GAbNmywhg0b2i233OL1Hv/880+7+eab7Z133rHvvvvONm7caK1atbLmzZt7rKMw93i51zDT4sWLrUmTJhYZGWnPP/+8xzIn97dnzx4rXbq0PfHEE7Z9+3b76aef7OOPP7bDhw8Xif4efPBBq1Gjhq1Zs8YSExPtlVdeseLFi9uSJUsc0d/lFMkwc7Hk5GSTZOvWrTMzs4yMDIuIiLApU6a4x6SlpVlYWJjNmTPHzM5/OPn6+trChQvdY3799VcrVqyYffrpp9e2gRykpqZarVq1LD4+3qKjo91hpij0N2rUKGvbtm2Oy53eY/fu3e3+++/3mNezZ0/r16+fmTm7v4s/SAuql2+++cYk2aZNm9xjNm7caJLsu+++u8pdebrUH/tMmzdvNkm2b98+M3NWjzn198svv1jFihVt165dVrVqVY8w4/T+evXq5f79y47T+2vQoIE9/fTTHvOaNWtmTz31lJk5q7/sFMnDTBc7duyYJKl06dKSpMTERCUlJalTp07uMf7+/oqOjtaGDRskSdu2bdOZM2c8xkRGRqphw4buMd42dOhQde/eXTfffLPH/KLQ39KlS9WiRQvdfffdKl++vK677jq99tpr7uVO77Ft27ZatWqVfvjhB0nS119/rfXr16tbt26SnN/fhQqql40bNyosLEytWrVyj7nhhhsUFhZWqPrNdOzYMblcLvd3wzm9x4yMDPXv319PPPGEGjRokGW5k/vLyMjQsmXLVLt2bXXu3Fnly5dXq1atPA7VOLk/6fxnztKlS/Xrr7/KzLRmzRr98MMP6ty5syTn91fkw4yZaeTIkWrbtq0aNmwoSe5v4b74m7fDw8Pdy5KSkuTn56dSpUrlOMabFi5cqO3bt2vy5MlZlhWF/n7++WfNnj1btWrV0meffabBgwfrkUce0RtvvCHJ+T2OGjVKvXv3Vt26deXr66vrrrtOsbGx6t27tyTn93ehguolKSlJ5cuXz7L+8uXLF6p+JSktLU2jR49Wnz593F/c5/Qep06dKh8fHz3yyCPZLndyf8nJyTp+/LimTJmiLl26aMWKFbrjjjvUs2dPrVu3TpKz+5OkF198UfXr11elSpXk5+enLl26aNasWWrbtq0k5/fniK8zuBLDhg3Tjh07tH79+izLXC6Xx7SZZZl3sdyMudoOHDigESNGaMWKFQoICMhxnFP7k87/T6lFixaaNGmSJOm6667T7t27NXv2bN13333ucU7t8Z133tGbb76pBQsWqEGDBkpISFBsbKwiIyM1YMAA9zin9pedguglu/GFrd8zZ87o3nvvVUZGhmbNmnXZ8U7ocdu2bXrhhRe0ffv2PNfhhP4yT7y//fbb9eijj0qSmjZtqg0bNmjOnDmKjo7O8bFO6E86H2Y2bdqkpUuXqmrVqvr88881ZMgQVahQIcve/Qs5pb8ivWdm+PDhWrp0qdasWaNKlSq550dEREhSliSZnJzs/t9jRESETp8+raNHj+Y4xlu2bdum5ORkNW/eXD4+PvLx8dG6dev04osvysfHx12fU/uTpAoVKqh+/foe8+rVq+e+ssDpr+ETTzyh0aNH695771WjRo3Uv39/Pfroo+49bU7v70IF1UtERIQOHz6cZf2//fZboen3zJkzuueee5SYmKj4+Hj3XhnJ2T1+8cUXSk5OVpUqVdyfOfv27dNjjz2mqKgoSc7ur2zZsvLx8bnsZ45T+zt16pT+/ve/a/r06br11lvVuHFjDRs2TL169dKzzz4rydn9SUU0zJiZhg0bpkWLFmn16tWqVq2ax/Jq1aopIiJC8fHx7nmnT5/WunXr1KZNG0lS8+bN5evr6zHm0KFD2rVrl3uMt3To0EE7d+5UQkKC+6dFixbq27evEhISVL16dUf3J0l/+ctfslxO/8MPP7i/Ld3pr+HJkydVrJjnr1/x4sXd/0N0en8XKqheWrdurWPHjmnz5s3uMf/973917NixQtFvZpD58ccftXLlSpUpU8ZjuZN77N+/v3bs2OHxmRMZGaknnnhCn332mSRn9+fn56eWLVte8jPHyf2dOXNGZ86cueRnjpP7k1Q0L81++OGHLSwszNauXWuHDh1y/5w8edI9ZsqUKRYWFmaLFi2ynTt3Wu/evbO9VLRSpUq2cuVK2759u7Vv375QXNabnQuvZjJzfn+bN282Hx8fmzhxov3444/21ltvWVBQkL355pvuMU7uccCAAVaxYkX3pdmLFi2ysmXL2pNPPuke46T+UlNT7auvvrKvvvrKJNn06dPtq6++cl/JU1C9dOnSxRo3bmwbN260jRs3WqNGja7ZZaGX6vHMmTN22223WaVKlSwhIcHjcyc9Pd0RPV7uNbzYxVczmTm7v0WLFpmvr6+9+uqr9uOPP9pLL71kxYsXty+++KJI9BcdHW0NGjSwNWvW2M8//2xz5861gIAAmzVrliP6u5wiGWYkZfszd+5c95iMjAwbP368RUREmL+/v9100022c+dOj/WcOnXKhg0bZqVLl7bAwEC75ZZbbP/+/de4m9y5OMwUhf4++ugja9iwofn7+1vdunXt1Vdf9Vju5B5TUlJsxIgRVqVKFQsICLDq1avb2LFjPf7wOam/NWvWZPs7N2DAgALt5ciRI9a3b18LDQ210NBQ69u3rx09etTrPSYmJub4ubNmzRpH9Hi51/Bi2YUZp/f373//22rWrGkBAQHWpEkTj3uwOL2/Q4cO2cCBAy0yMtICAgKsTp069txzz1lGRoYj+rscl5nZ1drrAwAAcLUVyXNmAADA/w7CDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAAcDTCDAAUMi6XS0uWLPF2GYBjEGaAIig5OVmDBg1SlSpV5O/vr4iICHXu3FkbN270dmmFRmEIDHFxcWratKlXawCKAh9vFwCg4N155506c+aM5s+fr+rVq+vw4cNatWqV/vjjD2+XBgAFjj0zQBHz559/av369Zo6dapiYmJUtWpVXX/99RozZoy6d+/uHnfs2DE99NBDKl++vEqUKKH27dvr66+/9ljXlClTFB4ertDQUD3wwAMaPXq0x56Edu3aKTY21uMxPXr00MCBA93Tp0+f1pNPPqmKFSsqODhYrVq10tq1a93L582bp5IlS+qzzz5TvXr1FBISoi5duujQoUMe63399dfVoEED+fv7q0KFCho2bFieesmruXPnql69egoICFDdunU1a9Ys97K9e/fK5XJp0aJFiomJUVBQkJo0aZJlz9drr72mypUrKygoSHfccYemT5+ukiVLuvueMGGCvv76a7lcLrlcLs2bN8/92N9//1133HGHgoKCVKtWLS1duvSK+gGKMsIMUMSEhIQoJCRES5YsUXp6erZjzEzdu3dXUlKSli9frm3btqlZs2bq0KGDe+/Nu+++q/Hjx2vixInaunWrKlSo4PEHPbf++te/6ssvv9TChQu1Y8cO3X333erSpYt+/PFH95iTJ0/q2Wef1X/+8x99/vnn2r9/vx5//HH38tmzZ2vo0KF66KGHtHPnTi1dulQ1a9bMdS959dprr2ns2LGaOHGivv32W02aNEn/+Mc/NH/+fI9xY8eO1eOPP66EhATVrl1bvXv31tmzZyVJX375pQYPHqwRI0YoISFBHTt21MSJE92P7dWrlx577DE1aNBAhw4d0qFDh9SrVy/38gkTJuiee+7Rjh071K1bN/Xt25c9a0BOvPs9lwCuhvfff99KlSplAQEB1qZNGxszZox9/fXX7uWrVq2yEiVKWFpamsfjatSoYa+88oqZmbVu3doGDx7ssbxVq1bWpEkT9/TF39ZuZnb77be7v6l3z5495nK57Ndff/UY06FDBxszZoyZmc2dO9ck2Z49e9zLX375ZQsPD3dPR0ZG2tixY7PtNTe9ZEeSLV68ONtllStXtgULFnjM++c//2mtW7c2M3N/S/a//vUv9/Ldu3ebJPv222/NzKxXr17WvXt3j3X07dvXwsLC3NPjx4/3eD4vrO2pp55yTx8/ftxcLpd98sknOfYD/C9jzwxQBN155506ePCgli5dqs6dO2vt2rVq1qyZ+zDGtm3bdPz4cZUpU8a9JyckJESJiYn66aefJEnffvutWrdu7bHei6cvZ/v27TIz1a5d22M769atc29HkoKCglSjRg33dIUKFZScnCzp/MnMBw8eVIcOHbLdRm56yYvffvtNBw4c0AMPPOCxvmeeeSbL+ho3buxRc2a9kvT999/r+uuv9xh/8fSlXLju4OBghYaGutcNwBMnAANFVEBAgDp27KiOHTtq3LhxevDBBzV+/HgNHDhQGRkZqlChgse5K5kyz+nIjWLFisnMPOadOXPG/e+MjAwVL15c27ZtU/HixT3GhYSEuP/t6+vrsczlcrnXGxgYeMkaCqqXC9cnnT/U1KpVK49lF/dwYd0ul8vj8Wbmnpfp4ufqUrJ7TjLXDcATYQb4H1G/fn33pcjNmjVTUlKSfHx8FBUVle34evXqadOmTbrvvvvc8zZt2uQxply5ch4n6p47d067du1STEyMJOm6667TuXPnlJycrBtvvDFfdYeGhioqKkqrVq1yr/dCueklL8LDw1WxYkX9/PPP6tu3b77XU7duXW3evNlj3tatWz2m/fz8dO7cuXxvA8B5hBmgiDly5Ijuvvtu3X///WrcuLFCQ0O1detWTZs2Tbfffrsk6eabb1br1q3Vo0cPTZ06VXXq1NHBgwe1fPly9ejRQy1atNCIESM0YMAAtWjRQm3bttVbb72l3bt3q3r16u5ttW/fXiNHjtSyZctUo0YNPf/88/rzzz/dy2vXrq2+ffvqvvvu03PPPafrrrtOv//+u1avXq1GjRqpW7duueopLi5OgwcPVvny5dW1a1elpqbqyy+/1PDhw3PVS04SExOVkJDgMa9mzZqKi4vTI488ohIlSqhr165KT0/X1q1bdfToUY0cOTJXNQ8fPlw33XSTpk+frltvvVWrV6/WJ5984rG3Jioqyl1DpUqVFBoaKn9//1ytH8AFvHrGDoACl5aWZqNHj7ZmzZpZWFiYBQUFWZ06deypp56ykydPuselpKTY8OHDLTIy0nx9fa1y5crWt29f279/v3vMxIkTrWzZshYSEmIDBgywJ5980uOE1dOnT9vDDz9spUuXtvLly9vkyZM9TgDOHDNu3DiLiooyX19fi4iIsDvuuMN27NhhZudPAL7wpFgzs8WLF9vFH09z5syxOnXqmK+vr1WoUMGGDx+ep14uJinbnzVr1piZ2VtvvWVNmzY1Pz8/K1WqlN100022aNEiM/v/JwB/9dVX7vUdPXrU4/FmZq+++qpVrFjRAgMDrUePHvbMM89YRESEx2t15513WsmSJU2SzZ07113bxScnh4WFuZcD8OQyy8NBXAD/0+Li4rRkyZIsezOQO3/729/03Xff6YsvvvB2KUCRwmEmALhKnn32WXXs2FHBwcH65JNPNH/+/HzdqwfApRFmAOAq2bx5s6ZNm6bU1FRVr15dL774oh588EFvlwUUORxmAgAAjsZN8wAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKMRZgAAgKP9P3zrJYLcYqeBAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Max Sequence Length: 1854\n",
      "Min Sequence Length: 249\n",
      "The length of selected data: 500\n"
     ]
    }
   ],
   "source": [
    "def plot_sequence_lengths(data, split=\"train\", max_length=2048):\n",
    "    sequence_lengths = []\n",
    "    selected_indices = []\n",
    "    \n",
    "    for idx, example in tqdm(enumerate(data), total=len(data)):\n",
    "        sequence_lengths.append(len(example[\"text\"]))\n",
    "        if sequence_lengths[idx] < max_length:\n",
    "            selected_indices.append(idx)\n",
    "\n",
    "    plt.hist(sequence_lengths, bins=30)\n",
    "    plt.xlabel(\"Sequence Length\")\n",
    "    plt.ylabel(\"Count\")\n",
    "    plt.title(\"Distribution of Text Sequence Lengths\")\n",
    "    plt.show()\n",
    "    \n",
    "    print(\"Max Sequence Length:\", max(sequence_lengths))\n",
    "    print(\"Min Sequence Length:\", min(sequence_lengths))\n",
    "\n",
    "    return selected_indices\n",
    "\n",
    "keep_indices_train = plot_sequence_lengths(data)\n",
    "data = data.select(keep_indices_train)\n",
    "print(\"The length of selected data:\", len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b72beda5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:17:52.700177Z",
     "iopub.status.busy": "2023-08-09T06:17:52.699241Z",
     "iopub.status.idle": "2023-08-09T06:20:06.695612Z",
     "shell.execute_reply": "2023-08-09T06:20:06.693425Z"
    },
    "papermill": {
     "duration": 134.013913,
     "end_time": "2023-08-09T06:20:06.698400",
     "exception": false,
     "start_time": "2023-08-09T06:17:52.684487",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3bb42326dd2444abec7b293fc0e1003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/200 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bced088ed514fdfb8d94b2445da1abc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/1.46k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1714: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9098f78ae1424b9e98e16d2923ccb846",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/798k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbb0b79d250244b481f4b9d3f48122a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cdc55a091884a79a69921c969ad62ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/90.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:2193: FutureWarning: The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26f849ff89c24c84817a85d8424f340b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading model.safetensors:   0%|          | 0.00/10.7G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of GPTNeoForCausalLM were not initialized from the model checkpoint at EleutherAI/gpt-neo-2.7B and are newly initialized: ['lm_head.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPTNeoForCausalLM(\n",
      "  (transformer): GPTNeoModel(\n",
      "    (wte): Embedding(50257, 2560)\n",
      "    (wpe): Embedding(2048, 2560)\n",
      "    (drop): Dropout(p=0.0, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-31): 32 x GPTNeoBlock(\n",
      "        (ln_1): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPTNeoAttention(\n",
      "          (attention): GPTNeoSelfAttention(\n",
      "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
      "            (resid_dropout): Dropout(p=0.0, inplace=False)\n",
      "            (k_proj): Linear4bit(in_features=2560, out_features=2560, bias=False)\n",
      "            (v_proj): Linear4bit(in_features=2560, out_features=2560, bias=False)\n",
      "            (q_proj): Linear4bit(in_features=2560, out_features=2560, bias=False)\n",
      "            (out_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n",
      "          )\n",
      "        )\n",
      "        (ln_2): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPTNeoMLP(\n",
      "          (c_fc): Linear4bit(in_features=2560, out_features=10240, bias=True)\n",
      "          (c_proj): Linear4bit(in_features=10240, out_features=2560, bias=True)\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(in_features=2560, out_features=50257, bias=False)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "model_id = \"EleutherAI/gpt-neo-2.7B\"\n",
    "access_token = \"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\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    model_id, \n",
    "    use_auth_token=access_token, \n",
    "    trust_remote_code=True,\n",
    "#     pad_token=\"<|endoftext|>\"\n",
    ")\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_use_double_quant=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=torch.bfloat16\n",
    ")\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    quantization_config=bnb_config,\n",
    "    trust_remote_code=True,\n",
    "    use_auth_token=access_token\n",
    ")\n",
    "\n",
    "model.config.use_cache=False\n",
    "# model.config.pretraining_tp=1\n",
    "\n",
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "68367451",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:20:06.735800Z",
     "iopub.status.busy": "2023-08-09T06:20:06.735410Z",
     "iopub.status.idle": "2023-08-09T06:20:06.742051Z",
     "shell.execute_reply": "2023-08-09T06:20:06.740984Z"
    },
    "papermill": {
     "duration": 0.028768,
     "end_time": "2023-08-09T06:20:06.744470",
     "exception": false,
     "start_time": "2023-08-09T06:20:06.715702",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "qlora_config = LoraConfig(\n",
    "    r=64,\n",
    "    lora_alpha=128,\n",
    "    lora_dropout=0.05,\n",
    "    bias=\"none\",\n",
    "#     target_modules=[\"W_pack\", \"o_proj\", \"gate_proj\", \"down_proj\", \"up_proj\"],\n",
    "#     target_modules=[\"c_attn\", \"c_proj\", \"w1\", \"w2\"],\n",
    "    target_modules=[\"k_proj\", \"v_proj\", \"q_proj\", \"out_proj\", \"c_fc\", \"c_proj\", \"lm_head\"], #EleutherAI/gpt-neo-2.7B\n",
    "    task_type=\"CAUSAL_LM\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3f9c565b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:20:06.781712Z",
     "iopub.status.busy": "2023-08-09T06:20:06.781008Z",
     "iopub.status.idle": "2023-08-09T06:20:34.162852Z",
     "shell.execute_reply": "2023-08-09T06:20:34.161841Z"
    },
    "papermill": {
     "duration": 27.403918,
     "end_time": "2023-08-09T06:20:34.165958",
     "exception": false,
     "start_time": "2023-08-09T06:20:06.762040",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 97752128 || all params: 1490768448 || trainable%: 6.557163731976168\n",
      "PeftModelForCausalLM(\n",
      "  (base_model): LoraModel(\n",
      "    (model): GPTNeoForCausalLM(\n",
      "      (transformer): GPTNeoModel(\n",
      "        (wte): Embedding(50257, 2560)\n",
      "        (wpe): Embedding(2048, 2560)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "        (h): ModuleList(\n",
      "          (0-31): 32 x GPTNeoBlock(\n",
      "            (ln_1): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "            (attn): GPTNeoAttention(\n",
      "              (attention): GPTNeoSelfAttention(\n",
      "                (attn_dropout): Dropout(p=0.0, inplace=False)\n",
      "                (resid_dropout): Dropout(p=0.0, inplace=False)\n",
      "                (k_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=False\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "                (v_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=False\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "                (q_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=False\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "                (out_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=True\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "              )\n",
      "            )\n",
      "            (ln_2): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): GPTNeoMLP(\n",
      "              (c_fc): Linear4bit(\n",
      "                in_features=2560, out_features=10240, bias=True\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=64, out_features=10240, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (c_proj): Linear4bit(\n",
      "                in_features=10240, out_features=2560, bias=True\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=10240, out_features=64, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (act): NewGELUActivation()\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "        (ln_f): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "      (lm_head): Linear(\n",
      "        in_features=2560, out_features=50257, bias=False\n",
      "        (lora_dropout): ModuleDict(\n",
      "          (default): Dropout(p=0.05, inplace=False)\n",
      "        )\n",
      "        (lora_A): ModuleDict(\n",
      "          (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "        )\n",
      "        (lora_B): ModuleDict(\n",
      "          (default): Linear(in_features=64, out_features=50257, bias=False)\n",
      "        )\n",
      "        (lora_embedding_A): ParameterDict()\n",
      "        (lora_embedding_B): ParameterDict()\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "def print_trainable_parameters(model):\n",
    "    trainable_params = 0\n",
    "    all_param = 0\n",
    "    for _, param in model.named_parameters():\n",
    "        all_param += param.numel()\n",
    "        if param.requires_grad:\n",
    "            trainable_params += param.numel()\n",
    "    print(\n",
    "        f'trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}'\n",
    "    )\n",
    "\n",
    "model_ = get_peft_model(model, qlora_config)\n",
    "print_trainable_parameters(model_)\n",
    "print(model_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3edc2bf2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:20:34.200194Z",
     "iopub.status.busy": "2023-08-09T06:20:34.198476Z",
     "iopub.status.idle": "2023-08-09T06:20:35.750053Z",
     "shell.execute_reply": "2023-08-09T06:20:35.749099Z"
    },
    "papermill": {
     "duration": 1.570274,
     "end_time": "2023-08-09T06:20:35.752300",
     "exception": false,
     "start_time": "2023-08-09T06:20:34.182026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/peft/utils/other.py:102: FutureWarning: prepare_model_for_int8_training is deprecated and will be removed in a future version. Use prepare_model_for_kbit_training instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4df5e05562c47f8834f56417d2ca648",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./SFT-Train\", \n",
    "    per_device_train_batch_size=2,\n",
    "#     per_device_eval_batch_size=2,\n",
    "#     gradient_accumulation_steps=2,\n",
    "    learning_rate=5e-6,\n",
    "    logging_steps=20,\n",
    "    logging_strategy=\"steps\",\n",
    "    save_steps=500,\n",
    "    save_strategy=\"steps\",\n",
    "    eval_steps=100,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    save_total_limit=1,\n",
    "    load_best_model_at_end=True,\n",
    "    max_steps=2,\n",
    "    num_train_epochs=2,\n",
    "    optim=\"paged_adamw_8bit\",\n",
    "    fp16=True,\n",
    "    run_name=\"baseline-sft\",\n",
    "    report_to=\"none\"\n",
    ")\n",
    "\n",
    "response_template_with_context = \"### Answer:\"\n",
    "# response_template_ids = tokenizer.encode(response_template_with_context, add_special_tokens=False)[2:]\n",
    "# response_template_text = tokenizer.decode(response_template_ids)\n",
    "# print(response_template_ids)\n",
    "data_collator = DataCollatorForCompletionOnlyLM(response_template_with_context, tokenizer=tokenizer)\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model,\n",
    "    train_dataset=data,\n",
    "    args=training_args,\n",
    "    tokenizer=tokenizer,\n",
    "    peft_config=qlora_config,\n",
    "    dataset_text_field=\"text\",\n",
    "    max_seq_length=2048,\n",
    "    data_collator=data_collator,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f66c31f5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:20:35.785514Z",
     "iopub.status.busy": "2023-08-09T06:20:35.785222Z",
     "iopub.status.idle": "2023-08-09T06:20:59.138816Z",
     "shell.execute_reply": "2023-08-09T06:20:59.137861Z"
    },
    "papermill": {
     "duration": 23.372232,
     "end_time": "2023-08-09T06:20:59.140785",
     "exception": false,
     "start_time": "2023-08-09T06:20:35.768553",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a GPT2TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='2' max='2' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [2/2 00:09, Epoch 0/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2, training_loss=2.1079678535461426, metrics={'train_runtime': 22.885, 'train_samples_per_second': 0.35, 'train_steps_per_second': 0.087, 'total_flos': 22078950368256.0, 'train_loss': 2.1079678535461426, 'epoch': 0.02})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "085beb3b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:20:59.175454Z",
     "iopub.status.busy": "2023-08-09T06:20:59.174552Z",
     "iopub.status.idle": "2023-08-09T06:20:59.929969Z",
     "shell.execute_reply": "2023-08-09T06:20:59.929031Z"
    },
    "papermill": {
     "duration": 0.775449,
     "end_time": "2023-08-09T06:20:59.932966",
     "exception": false,
     "start_time": "2023-08-09T06:20:59.157517",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving The Final Model...\n"
     ]
    }
   ],
   "source": [
    "print(\"Saving The Final Model...\")\n",
    "trainer.save_model(\"./finetuned_model_adapter\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "dcfb5fe8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:20:59.974610Z",
     "iopub.status.busy": "2023-08-09T06:20:59.974265Z",
     "iopub.status.idle": "2023-08-09T06:22:25.968013Z",
     "shell.execute_reply": "2023-08-09T06:22:25.966036Z"
    },
    "papermill": {
     "duration": 86.015293,
     "end_time": "2023-08-09T06:22:25.970291",
     "exception": false,
     "start_time": "2023-08-09T06:20:59.954998",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of GPTNeoForCausalLM were not initialized from the model checkpoint at EleutherAI/gpt-neo-2.7B and are newly initialized: ['lm_head.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PeftModelForCausalLM(\n",
      "  (base_model): LoraModel(\n",
      "    (model): GPTNeoForCausalLM(\n",
      "      (transformer): GPTNeoModel(\n",
      "        (wte): Embedding(50257, 2560)\n",
      "        (wpe): Embedding(2048, 2560)\n",
      "        (drop): Dropout(p=0.0, inplace=False)\n",
      "        (h): ModuleList(\n",
      "          (0-31): 32 x GPTNeoBlock(\n",
      "            (ln_1): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "            (attn): GPTNeoAttention(\n",
      "              (attention): GPTNeoSelfAttention(\n",
      "                (attn_dropout): Dropout(p=0.0, inplace=False)\n",
      "                (resid_dropout): Dropout(p=0.0, inplace=False)\n",
      "                (k_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=False\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "                (v_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=False\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "                (q_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=False\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "                (out_proj): Linear4bit(\n",
      "                  in_features=2560, out_features=2560, bias=True\n",
      "                  (lora_dropout): ModuleDict(\n",
      "                    (default): Dropout(p=0.05, inplace=False)\n",
      "                  )\n",
      "                  (lora_A): ModuleDict(\n",
      "                    (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                  )\n",
      "                  (lora_B): ModuleDict(\n",
      "                    (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                  )\n",
      "                  (lora_embedding_A): ParameterDict()\n",
      "                  (lora_embedding_B): ParameterDict()\n",
      "                )\n",
      "              )\n",
      "            )\n",
      "            (ln_2): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "            (mlp): GPTNeoMLP(\n",
      "              (c_fc): Linear4bit(\n",
      "                in_features=2560, out_features=10240, bias=True\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=64, out_features=10240, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (c_proj): Linear4bit(\n",
      "                in_features=10240, out_features=2560, bias=True\n",
      "                (lora_dropout): ModuleDict(\n",
      "                  (default): Dropout(p=0.05, inplace=False)\n",
      "                )\n",
      "                (lora_A): ModuleDict(\n",
      "                  (default): Linear(in_features=10240, out_features=64, bias=False)\n",
      "                )\n",
      "                (lora_B): ModuleDict(\n",
      "                  (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "                )\n",
      "                (lora_embedding_A): ParameterDict()\n",
      "                (lora_embedding_B): ParameterDict()\n",
      "              )\n",
      "              (act): NewGELUActivation()\n",
      "              (dropout): Dropout(p=0.0, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "        (ln_f): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "      (lm_head): Linear(\n",
      "        in_features=2560, out_features=50257, bias=False\n",
      "        (lora_dropout): ModuleDict(\n",
      "          (default): Dropout(p=0.05, inplace=False)\n",
      "        )\n",
      "        (lora_A): ModuleDict(\n",
      "          (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "        )\n",
      "        (lora_B): ModuleDict(\n",
      "          (default): Linear(in_features=64, out_features=50257, bias=False)\n",
      "        )\n",
      "        (lora_embedding_A): ParameterDict()\n",
      "        (lora_embedding_B): ParameterDict()\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "finetuned_model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    quantization_config=bnb_config,\n",
    "    trust_remote_code=True,\n",
    "    use_auth_token=access_token\n",
    ")\n",
    "finetuned_model = PeftModel.from_pretrained(finetuned_model, \"./finetuned_model_adapter\")\n",
    "# finetuned_model = finetuned_model.merge_and_unload()\n",
    "print(finetuned_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e115ef1f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:22:26.007166Z",
     "iopub.status.busy": "2023-08-09T06:22:26.006830Z",
     "iopub.status.idle": "2023-08-09T06:22:26.028247Z",
     "shell.execute_reply": "2023-08-09T06:22:26.027220Z"
    },
    "papermill": {
     "duration": 0.045023,
     "end_time": "2023-08-09T06:22:26.032225",
     "exception": false,
     "start_time": "2023-08-09T06:22:25.987202",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPTNeoForCausalLM(\n",
      "  (transformer): GPTNeoModel(\n",
      "    (wte): Embedding(50257, 2560)\n",
      "    (wpe): Embedding(2048, 2560)\n",
      "    (drop): Dropout(p=0.0, inplace=False)\n",
      "    (h): ModuleList(\n",
      "      (0-31): 32 x GPTNeoBlock(\n",
      "        (ln_1): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): GPTNeoAttention(\n",
      "          (attention): GPTNeoSelfAttention(\n",
      "            (attn_dropout): Dropout(p=0.0, inplace=False)\n",
      "            (resid_dropout): Dropout(p=0.0, inplace=False)\n",
      "            (k_proj): Linear4bit(\n",
      "              in_features=2560, out_features=2560, bias=False\n",
      "              (lora_dropout): ModuleDict(\n",
      "                (default): Dropout(p=0.05, inplace=False)\n",
      "              )\n",
      "              (lora_A): ModuleDict(\n",
      "                (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "              )\n",
      "              (lora_B): ModuleDict(\n",
      "                (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "              )\n",
      "              (lora_embedding_A): ParameterDict()\n",
      "              (lora_embedding_B): ParameterDict()\n",
      "            )\n",
      "            (v_proj): Linear4bit(\n",
      "              in_features=2560, out_features=2560, bias=False\n",
      "              (lora_dropout): ModuleDict(\n",
      "                (default): Dropout(p=0.05, inplace=False)\n",
      "              )\n",
      "              (lora_A): ModuleDict(\n",
      "                (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "              )\n",
      "              (lora_B): ModuleDict(\n",
      "                (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "              )\n",
      "              (lora_embedding_A): ParameterDict()\n",
      "              (lora_embedding_B): ParameterDict()\n",
      "            )\n",
      "            (q_proj): Linear4bit(\n",
      "              in_features=2560, out_features=2560, bias=False\n",
      "              (lora_dropout): ModuleDict(\n",
      "                (default): Dropout(p=0.05, inplace=False)\n",
      "              )\n",
      "              (lora_A): ModuleDict(\n",
      "                (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "              )\n",
      "              (lora_B): ModuleDict(\n",
      "                (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "              )\n",
      "              (lora_embedding_A): ParameterDict()\n",
      "              (lora_embedding_B): ParameterDict()\n",
      "            )\n",
      "            (out_proj): Linear4bit(\n",
      "              in_features=2560, out_features=2560, bias=True\n",
      "              (lora_dropout): ModuleDict(\n",
      "                (default): Dropout(p=0.05, inplace=False)\n",
      "              )\n",
      "              (lora_A): ModuleDict(\n",
      "                (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "              )\n",
      "              (lora_B): ModuleDict(\n",
      "                (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "              )\n",
      "              (lora_embedding_A): ParameterDict()\n",
      "              (lora_embedding_B): ParameterDict()\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "        (ln_2): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): GPTNeoMLP(\n",
      "          (c_fc): Linear4bit(\n",
      "            in_features=2560, out_features=10240, bias=True\n",
      "            (lora_dropout): ModuleDict(\n",
      "              (default): Dropout(p=0.05, inplace=False)\n",
      "            )\n",
      "            (lora_A): ModuleDict(\n",
      "              (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "            )\n",
      "            (lora_B): ModuleDict(\n",
      "              (default): Linear(in_features=64, out_features=10240, bias=False)\n",
      "            )\n",
      "            (lora_embedding_A): ParameterDict()\n",
      "            (lora_embedding_B): ParameterDict()\n",
      "          )\n",
      "          (c_proj): Linear4bit(\n",
      "            in_features=10240, out_features=2560, bias=True\n",
      "            (lora_dropout): ModuleDict(\n",
      "              (default): Dropout(p=0.05, inplace=False)\n",
      "            )\n",
      "            (lora_A): ModuleDict(\n",
      "              (default): Linear(in_features=10240, out_features=64, bias=False)\n",
      "            )\n",
      "            (lora_B): ModuleDict(\n",
      "              (default): Linear(in_features=64, out_features=2560, bias=False)\n",
      "            )\n",
      "            (lora_embedding_A): ParameterDict()\n",
      "            (lora_embedding_B): ParameterDict()\n",
      "          )\n",
      "          (act): NewGELUActivation()\n",
      "          (dropout): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (ln_f): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (lm_head): Linear(\n",
      "    in_features=2560, out_features=50257, bias=False\n",
      "    (lora_dropout): ModuleDict(\n",
      "      (default): Dropout(p=0.05, inplace=False)\n",
      "    )\n",
      "    (lora_A): ModuleDict(\n",
      "      (default): Linear(in_features=2560, out_features=64, bias=False)\n",
      "    )\n",
      "    (lora_B): ModuleDict(\n",
      "      (default): Linear(in_features=64, out_features=50257, bias=False)\n",
      "    )\n",
      "    (lora_embedding_A): ParameterDict()\n",
      "    (lora_embedding_B): ParameterDict()\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ab779ee8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:22:26.068592Z",
     "iopub.status.busy": "2023-08-09T06:22:26.068303Z",
     "iopub.status.idle": "2023-08-09T06:22:26.627954Z",
     "shell.execute_reply": "2023-08-09T06:22:26.627000Z"
    },
    "papermill": {
     "duration": 0.581391,
     "end_time": "2023-08-09T06:22:26.630637",
     "exception": false,
     "start_time": "2023-08-09T06:22:26.049246",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "They are the same model\n"
     ]
    }
   ],
   "source": [
    "params1 = model.state_dict()\n",
    "params2 = finetuned_model.base_model.model.state_dict()\n",
    "\n",
    "def are_models_equal(params1, params2):\n",
    "    for key in params2.keys():\n",
    "        if key in params1.keys():\n",
    "            if not torch.allclose(params1[key].half(), params2[key].half()):\n",
    "                return False\n",
    "#             else:\n",
    "#                 print(params1[key].half())\n",
    "#                 print(params2[key].half())\n",
    "#                 print(\"Same!!!\")\n",
    "        else:\n",
    "            print(\"Additional Keys:\", key)\n",
    "    return True\n",
    "\n",
    "if are_models_equal(params1, params2):\n",
    "    print(\"They are the same model\")\n",
    "else:\n",
    "    print(\"They are not the same model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bf1d5249",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:22:26.670312Z",
     "iopub.status.busy": "2023-08-09T06:22:26.669972Z",
     "iopub.status.idle": "2023-08-09T06:22:26.712420Z",
     "shell.execute_reply": "2023-08-09T06:22:26.711392Z"
    },
    "papermill": {
     "duration": 0.065982,
     "end_time": "2023-08-09T06:22:26.714630",
     "exception": false,
     "start_time": "2023-08-09T06:22:26.648648",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prompt</th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "      <th>E</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Which of the following statements accurately d...</td>\n",
       "      <td>MOND is a theory that reduces the observed mis...</td>\n",
       "      <td>MOND is a theory that increases the discrepanc...</td>\n",
       "      <td>MOND is a theory that explains the missing bar...</td>\n",
       "      <td>MOND is a theory that reduces the discrepancy ...</td>\n",
       "      <td>MOND is a theory that eliminates the observed ...</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Which of the following is an accurate definiti...</td>\n",
       "      <td>Dynamic scaling refers to the evolution of sel...</td>\n",
       "      <td>Dynamic scaling refers to the non-evolution of...</td>\n",
       "      <td>Dynamic scaling refers to the evolution of sel...</td>\n",
       "      <td>Dynamic scaling refers to the non-evolution of...</td>\n",
       "      <td>Dynamic scaling refers to the evolution of sel...</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Which of the following statements accurately d...</td>\n",
       "      <td>The triskeles symbol was reconstructed as a fe...</td>\n",
       "      <td>The triskeles symbol is a representation of th...</td>\n",
       "      <td>The triskeles symbol is a representation of a ...</td>\n",
       "      <td>The triskeles symbol represents three interloc...</td>\n",
       "      <td>The triskeles symbol is a representation of th...</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is the significance of regularization in ...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>Regularizing the mass-energy of an electron wi...</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Which of the following statements accurately d...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>The angular spacing of features in the diffrac...</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               prompt  \\\n",
       "id                                                      \n",
       "0   Which of the following statements accurately d...   \n",
       "1   Which of the following is an accurate definiti...   \n",
       "2   Which of the following statements accurately d...   \n",
       "3   What is the significance of regularization in ...   \n",
       "4   Which of the following statements accurately d...   \n",
       "\n",
       "                                                    A  \\\n",
       "id                                                      \n",
       "0   MOND is a theory that reduces the observed mis...   \n",
       "1   Dynamic scaling refers to the evolution of sel...   \n",
       "2   The triskeles symbol was reconstructed as a fe...   \n",
       "3   Regularizing the mass-energy of an electron wi...   \n",
       "4   The angular spacing of features in the diffrac...   \n",
       "\n",
       "                                                    B  \\\n",
       "id                                                      \n",
       "0   MOND is a theory that increases the discrepanc...   \n",
       "1   Dynamic scaling refers to the non-evolution of...   \n",
       "2   The triskeles symbol is a representation of th...   \n",
       "3   Regularizing the mass-energy of an electron wi...   \n",
       "4   The angular spacing of features in the diffrac...   \n",
       "\n",
       "                                                    C  \\\n",
       "id                                                      \n",
       "0   MOND is a theory that explains the missing bar...   \n",
       "1   Dynamic scaling refers to the evolution of sel...   \n",
       "2   The triskeles symbol is a representation of a ...   \n",
       "3   Regularizing the mass-energy of an electron wi...   \n",
       "4   The angular spacing of features in the diffrac...   \n",
       "\n",
       "                                                    D  \\\n",
       "id                                                      \n",
       "0   MOND is a theory that reduces the discrepancy ...   \n",
       "1   Dynamic scaling refers to the non-evolution of...   \n",
       "2   The triskeles symbol represents three interloc...   \n",
       "3   Regularizing the mass-energy of an electron wi...   \n",
       "4   The angular spacing of features in the diffrac...   \n",
       "\n",
       "                                                    E answer  \n",
       "id                                                            \n",
       "0   MOND is a theory that eliminates the observed ...      D  \n",
       "1   Dynamic scaling refers to the evolution of sel...      A  \n",
       "2   The triskeles symbol is a representation of th...      A  \n",
       "3   Regularizing the mass-energy of an electron wi...      C  \n",
       "4   The angular spacing of features in the diffrac...      D  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if os.getenv(\"KAGGLE_IS_COMPETITION_RERUN\"):\n",
    "    test = pd.read_csv(\"/kaggle/input/kaggle-llm-science-exam/test.csv\", index_col=\"id\")\n",
    "    test[\"answer\"] = \"A\"\n",
    "else:\n",
    "    test = pd.read_csv(\"/kaggle/input/kaggle-llm-science-exam/train.csv\", index_col=\"id\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f57088d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:22:26.752129Z",
     "iopub.status.busy": "2023-08-09T06:22:26.751515Z",
     "iopub.status.idle": "2023-08-09T06:22:26.759511Z",
     "shell.execute_reply": "2023-08-09T06:22:26.758346Z"
    },
    "papermill": {
     "duration": 0.028113,
     "end_time": "2023-08-09T06:22:26.761635",
     "exception": false,
     "start_time": "2023-08-09T06:22:26.733522",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Perplexity(nn.Module):\n",
    "    def __init__(self, reduce: bool = True):\n",
    "        super().__init__()\n",
    "        self.loss_fn = nn.CrossEntropyLoss()\n",
    "        self.reduce = reduce\n",
    "\n",
    "    def forward(self, logits, labels):\n",
    "        shift_logits = logits[..., :-1, :].contiguous()\n",
    "        shift_labels = labels[..., 1:].contiguous()\n",
    "\n",
    "        perplexity = []\n",
    "        for i in range(labels.shape[0]):\n",
    "            perplexity.append(self.loss_fn(shift_logits[i], shift_labels[i]))\n",
    "        perplexity = torch.stack(perplexity, dim=0)\n",
    "        if self.reduce:\n",
    "            perplexity = torch.mean(perplexity)\n",
    "        return perplexity \n",
    "    \n",
    "perp = Perplexity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c765cd53",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:22:26.796795Z",
     "iopub.status.busy": "2023-08-09T06:22:26.796527Z",
     "iopub.status.idle": "2023-08-09T06:25:28.111075Z",
     "shell.execute_reply": "2023-08-09T06:25:28.110053Z"
    },
    "papermill": {
     "duration": 181.335511,
     "end_time": "2023-08-09T06:25:28.114247",
     "exception": false,
     "start_time": "2023-08-09T06:22:26.778736",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35f5e7c7e3884c00ace01bcd97ea5850",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/200 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:31: UserWarning: None of the inputs have requires_grad=True. Gradients will be None\n",
      "  warnings.warn(\"None of the inputs have requires_grad=True. Gradients will be None\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5275\n"
     ]
    }
   ],
   "source": [
    "def precision_at_k(r, k):\n",
    "    assert k <= len(r)\n",
    "    assert k != 0\n",
    "    return sum(int(x) for x in r[:k]) / k\n",
    "\n",
    "def MAP_at_3(predictions, true_items):\n",
    "    U = len(predictions)\n",
    "    map_at_3 = 0.0\n",
    "    for u in range(U):\n",
    "        user_preds = predictions[u]\n",
    "        user_true = true_items[u]\n",
    "        user_results = [1 if item == user_true else 0 for item in user_preds]\n",
    "        for k in range(min(len(user_preds), 3)):\n",
    "            map_at_3 += precision_at_k(user_results, k + 1) * user_results[k]\n",
    "    return map_at_3 / U\n",
    "\n",
    "maps = []\n",
    "preds = []\n",
    "for idx, row in tqdm(test.iterrows(), total=len(test)):\n",
    "    with torch.no_grad():\n",
    "        cols = [\"A\", \"B\", \"C\", \"D\", \"E\"]\n",
    "        perps = []\n",
    "        samples = []\n",
    "        for col in cols:\n",
    "            samples.append(\"<|question|>\" + row[\"prompt\"] + \\\n",
    "                           \"</s><|answer|>\" + \"answer: \" + row[col])\n",
    "        inputs = tokenizer(samples, return_tensors=\"pt\", \n",
    "                           add_special_tokens=False, padding=True, \n",
    "                           truncation=True).to(\"cuda\")\n",
    "        output = model(input_ids=inputs[\"input_ids\"], \n",
    "                       attention_mask=inputs[\"attention_mask\"])\n",
    "        output = output.logits\n",
    "        labels = inputs[\"input_ids\"]\n",
    "        labels.masked_fill_(~inputs[\"attention_mask\"].bool(), -100)\n",
    "        for j in range(len(cols)):\n",
    "            p = perp(output[j].unsqueeze(0), labels[j].unsqueeze(0))\n",
    "            perps.append(p.detach().cpu())\n",
    "            \n",
    "        del inputs\n",
    "        del labels\n",
    "        del output\n",
    "        del p\n",
    "\n",
    "    perps = np.array(perps)\n",
    "        \n",
    "    predictions = [np.array(cols)[np.argsort(perps)]]\n",
    "    preds.append(predictions)\n",
    "    tp = [row.answer]\n",
    "    map = MAP_at_3(predictions, tp)\n",
    "    maps.append(map)\n",
    "print(np.mean(maps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d072b49f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:25:28.153215Z",
     "iopub.status.busy": "2023-08-09T06:25:28.151981Z",
     "iopub.status.idle": "2023-08-09T06:25:28.166530Z",
     "shell.execute_reply": "2023-08-09T06:25:28.165508Z"
    },
    "papermill": {
     "duration": 0.036203,
     "end_time": "2023-08-09T06:25:28.168746",
     "exception": false,
     "start_time": "2023-08-09T06:25:28.132543",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission = pd.read_csv(\"/kaggle/input/kaggle-llm-science-exam/sample_submission.csv\")\n",
    "submission[\"prediction\"] = [\" \".join(p[0][:3]) for p in preds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d66f763b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:25:28.206269Z",
     "iopub.status.busy": "2023-08-09T06:25:28.205465Z",
     "iopub.status.idle": "2023-08-09T06:25:28.216980Z",
     "shell.execute_reply": "2023-08-09T06:25:28.215956Z"
    },
    "papermill": {
     "duration": 0.032827,
     "end_time": "2023-08-09T06:25:28.219261",
     "exception": false,
     "start_time": "2023-08-09T06:25:28.186434",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>A D E</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>C E A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>B E C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>C B D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>D B E</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id prediction\n",
       "0   0      A D E\n",
       "1   1      C E A\n",
       "2   2      B E C\n",
       "3   3      C B D\n",
       "4   4      D B E"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "dd4cfb2f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-08-09T06:25:28.255880Z",
     "iopub.status.busy": "2023-08-09T06:25:28.255609Z",
     "iopub.status.idle": "2023-08-09T06:25:28.262326Z",
     "shell.execute_reply": "2023-08-09T06:25:28.261488Z"
    },
    "papermill": {
     "duration": 0.02685,
     "end_time": "2023-08-09T06:25:28.264604",
     "exception": false,
     "start_time": "2023-08-09T06:25:28.237754",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 613.131066,
   "end_time": "2023-08-09T06:25:32.423972",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-08-09T06:15:19.292906",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "02f486b763c74748bf9b370911f79165": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_46a1d2d973de4424b191b9dace1b05d5",
       "placeholder": "​",
       "style": "IPY_MODEL_0bf6a5155dc1401a8f88ed7def28cf42",
       "value": " 1/1 [00:00&lt;00:00,  3.16ba/s]"
      }
     },
     "0653ce5dd8a8418fbeed26224d4c9090": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "0bf6a5155dc1401a8f88ed7def28cf42": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "0f1f7a8bfc1a4b24bb4e1bfdfc53346e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "13e8e7e1d568414e8f80fae8ca67c598": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "13f96f2cb8f8467d90b88b3d5c5507dd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_2486718adf71429ca5b7799a7fba70ac",
       "placeholder": "​",
       "style": "IPY_MODEL_b2866ecaa27d477b9f4ac808dee3c8c0",
       "value": "Downloading (…)cial_tokens_map.json: 100%"
      }
     },
     "141fe23115e2430ab5cfc01dfc872811": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "169deda65b234c11841275e28412e352": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "197b724ef70e48fb95a0edf5f2aca235": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_efa20add3c1e4d3aaf0ebafdbcac1e73",
       "placeholder": "​",
       "style": "IPY_MODEL_e6ff1c7cba1447b9ae5510a3cd0584d6",
       "value": " 1.46k/1.46k [00:00&lt;00:00, 103kB/s]"
      }
     },
     "1a0182849d374b24ace92a3eebc8c28a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c2c16d89ab774f44888516787fd81201",
        "IPY_MODEL_3a13241e144f4b9f8594ff9bf40ffa14",
        "IPY_MODEL_3e6b1f32098c49e59ef3b1fd74d9d383"
       ],
       "layout": "IPY_MODEL_32cf3fe68d1244cc93eeba295d4b3722"
      }
     },
     "1b5febbf5c2c43fd93de5a0a5b9a193b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "208244d504f44e62a76ae1286a6e66ad": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2486718adf71429ca5b7799a7fba70ac": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "26b4239bb27d4367b3739fd44298b69d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "26f849ff89c24c84817a85d8424f340b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_891be481a3f34cdf9f22b593a31e9b6a",
        "IPY_MODEL_d8433b0a7b794d9ba9d7eaa1feed5653",
        "IPY_MODEL_ea3ed888dfc04aa0abe6efa8ded27b13"
       ],
       "layout": "IPY_MODEL_68f7524f9d314ec3a4c059099c4fd378"
      }
     },
     "2a94484da7fa4c5fa9a61e4559f067f2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2bcc3f7b5ca24c88bf76ee8e5826cc44": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_403fe954b90f47cfaef57fef7c421567",
        "IPY_MODEL_bb809c5eff8a4e73a7cc1f6ef57b9865",
        "IPY_MODEL_c75bd6d7e6fb435b8d7303c280e10185"
       ],
       "layout": "IPY_MODEL_ff06c65b4e2141b4a013f9b09b8e0ba3"
      }
     },
     "32cf3fe68d1244cc93eeba295d4b3722": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "333e5ff08c474134a0cfcc1d9720eded": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c609578d923742bf99c74029251db9cd",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_48efa22158f542cba7f4703e196c7621",
       "value": 1.0
      }
     },
     "34f84742e2504e7c95752599c346859f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "35f5e7c7e3884c00ace01bcd97ea5850": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_755b29a59ff54c079f14066e31b769ad",
        "IPY_MODEL_ab89399ed40d47c291750cfe40ae783c",
        "IPY_MODEL_c736437bb8214fd9934da4896997653e"
       ],
       "layout": "IPY_MODEL_9a8d4e7065a44bb8838dd6f71034df5f"
      }
     },
     "38f20ba38aba4847ae77a6ac2d64503e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "39098750902448cf86250416f5906829": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3a13241e144f4b9f8594ff9bf40ffa14": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dc634211edb44ffaa56266090a98f742",
       "max": 500.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2a94484da7fa4c5fa9a61e4559f067f2",
       "value": 500.0
      }
     },
     "3cdc55a091884a79a69921c969ad62ba": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_13f96f2cb8f8467d90b88b3d5c5507dd",
        "IPY_MODEL_cbbf2f4309434192b4dec22e5cfd0a2b",
        "IPY_MODEL_85681feee8db416dbb4ba2e34e39cdfd"
       ],
       "layout": "IPY_MODEL_6f0c03048db24d94b96aa56ea271e44d"
      }
     },
     "3e60ecfc515f4bd49362d93116173dec": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3e6b1f32098c49e59ef3b1fd74d9d383": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a3c8e58e02d0412a8dc3e612e9dc3aa1",
       "placeholder": "​",
       "style": "IPY_MODEL_ed83c83fe8254075869f67ae1e888c1d",
       "value": " 500/500 [00:00&lt;00:00, 4783.64ex/s]"
      }
     },
     "3f19342be98848e0889a1f9ad205ba02": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_141fe23115e2430ab5cfc01dfc872811",
       "max": 456356.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_3f93a5f270ac4db7bc754aa0e9c05b60",
       "value": 456356.0
      }
     },
     "3f93a5f270ac4db7bc754aa0e9c05b60": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "403fe954b90f47cfaef57fef7c421567": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_26b4239bb27d4367b3739fd44298b69d",
       "placeholder": "​",
       "style": "IPY_MODEL_437bfbf2e6e343179928b9a5aadb0047",
       "value": "Extracting data files: 100%"
      }
     },
     "42008ceaf47f42d88d5c42e0f2dfe0d3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "437bfbf2e6e343179928b9a5aadb0047": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "43c1cf1a1d2943c99489885857cc48de": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "44e276891b0648f3bf9885d6865fe368": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4536b2b149cb4f319e6d4beac14343b8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "46a1d2d973de4424b191b9dace1b05d5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "483e0d4b53d6443ab9157f1577104a01": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "48863d8d2c7d4f098f3ff9026b257bfa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_604bb9ad6448413da2a9dd776a2a72a7",
        "IPY_MODEL_333e5ff08c474134a0cfcc1d9720eded",
        "IPY_MODEL_a3ed720903cc462bbbcdb5f22f9a649c"
       ],
       "layout": "IPY_MODEL_b7aa97505419402b8e4bb004d03b25bd"
      }
     },
     "48efa22158f542cba7f4703e196c7621": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "4a269e92939947119d3cc33e4b0a9380": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4a973c2b1b72498f879f93511578494f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4be62dbcff2b4e48956775284a743d7b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_cce73ae2cdf24ad5a4f0cc933b1681c7",
       "placeholder": "​",
       "style": "IPY_MODEL_bdfeb000777a494bb8f3ca56a643b88c",
       "value": " 500/500 [00:00&lt;00:00, 5505.59it/s]"
      }
     },
     "55b9e9be1f78445facbc1ac028e6a6d8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_807cf7e509d6434596c61f2e3329ebc4",
       "max": 500.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_777dde6780ff4ce5800e6b0ce6a485a1",
       "value": 500.0
      }
     },
     "56a016ec42b5449ba91d8d7e3953a2a6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "604bb9ad6448413da2a9dd776a2a72a7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3e60ecfc515f4bd49362d93116173dec",
       "placeholder": "​",
       "style": "IPY_MODEL_b4bd78c463c64c578a47e694568be5cd",
       "value": "Downloading data files: 100%"
      }
     },
     "652962c0a0404571b3e77019becaa951": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9cffb3bb7f8e435d93754a4309680b79",
       "max": 200.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_0653ce5dd8a8418fbeed26224d4c9090",
       "value": 200.0
      }
     },
     "681c68387c2644f2873b44cbb9bbd20b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "684582618bf64dd69ca566ce365f87b3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "68f7524f9d314ec3a4c059099c4fd378": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6a863da47263470f99152ca018d607d0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6d084275d0624a1b8553f7a4e28856d3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6f0c03048db24d94b96aa56ea271e44d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "735b65243d784ba4ad560bc12662023b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "755b29a59ff54c079f14066e31b769ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_43c1cf1a1d2943c99489885857cc48de",
       "placeholder": "​",
       "style": "IPY_MODEL_169deda65b234c11841275e28412e352",
       "value": "100%"
      }
     },
     "760dd23442c4474284af76a04d07f921": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8459f9c9ed7b4d54bd4578fc5cc2c52f",
       "max": 798156.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_93883a190365450c90721b0387724fb9",
       "value": 798156.0
      }
     },
     "777dde6780ff4ce5800e6b0ce6a485a1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "7aaec1a022ea498288ba9388d6d0a7cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ffbd928dfb1742be850baacbea258bca",
       "placeholder": "​",
       "style": "IPY_MODEL_f6d42ddaffc545d39d6074c844e0eba2",
       "value": "100%"
      }
     },
     "807cf7e509d6434596c61f2e3329ebc4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "810d8c61866a4422906e88f5feb11fde": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_208244d504f44e62a76ae1286a6e66ad",
       "placeholder": "​",
       "style": "IPY_MODEL_735b65243d784ba4ad560bc12662023b",
       "value": " 456k/456k [00:00&lt;00:00, 26.8MB/s]"
      }
     },
     "82cca1149aca4d7493773bd632c5a81d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8459f9c9ed7b4d54bd4578fc5cc2c52f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8461fe7c0381488593e2b990d0cb4cc1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "84c0ea02c4764072976d2d46f35e6c5d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6d084275d0624a1b8553f7a4e28856d3",
       "placeholder": "​",
       "style": "IPY_MODEL_9c999da3e79348a783ad6d804f129db4",
       "value": "Downloading (…)lve/main/config.json: 100%"
      }
     },
     "85681feee8db416dbb4ba2e34e39cdfd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8e06168c42c8405e957a2fabd36216aa",
       "placeholder": "​",
       "style": "IPY_MODEL_681c68387c2644f2873b44cbb9bbd20b",
       "value": " 90.0/90.0 [00:00&lt;00:00, 6.46kB/s]"
      }
     },
     "85e65e03b6b44d7a8ee04bb6fee3bfbf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "88d056b2043644c0b06b345eb23d7665": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "891be481a3f34cdf9f22b593a31e9b6a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4a269e92939947119d3cc33e4b0a9380",
       "placeholder": "​",
       "style": "IPY_MODEL_9e01a44d5e3443418ad128cb31ab9eca",
       "value": "Downloading model.safetensors: 100%"
      }
     },
     "8bced088ed514fdfb8d94b2445da1abc": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_84c0ea02c4764072976d2d46f35e6c5d",
        "IPY_MODEL_d63b253c60f1467791a0f6ab1a7353d6",
        "IPY_MODEL_197b724ef70e48fb95a0edf5f2aca235"
       ],
       "layout": "IPY_MODEL_ff1791d260e848f791297344ab6aaa8d"
      }
     },
     "8e06168c42c8405e957a2fabd36216aa": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9098f78ae1424b9e98e16d2923ccb846": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_e8a15dc4b6924a8f82c1b58d003a1be7",
        "IPY_MODEL_760dd23442c4474284af76a04d07f921",
        "IPY_MODEL_9efa010a723d490e982122237b3839c4"
       ],
       "layout": "IPY_MODEL_c479ea86880c480fac79eb26b7bb376e"
      }
     },
     "910e16a1c8114b7489415836277e5edd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6a863da47263470f99152ca018d607d0",
       "placeholder": "​",
       "style": "IPY_MODEL_fc45d24fee47408bb8d38e8abcf2003a",
       "value": " 200/200 [00:00&lt;00:00, 13.1kB/s]"
      }
     },
     "920982e62da44f799b7fc6daf813ebaf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "931632e5b2a54c439d7314cc4fe0b8bf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "93883a190365450c90721b0387724fb9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "95f92684e9b5414aabf27ee6e47af5cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "9a8d4e7065a44bb8838dd6f71034df5f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9c999da3e79348a783ad6d804f129db4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "9cffb3bb7f8e435d93754a4309680b79": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9e01a44d5e3443418ad128cb31ab9eca": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "9efa010a723d490e982122237b3839c4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_df807131752f4a6b905bdecbf9922f97",
       "placeholder": "​",
       "style": "IPY_MODEL_13e8e7e1d568414e8f80fae8ca67c598",
       "value": " 798k/798k [00:00&lt;00:00, 8.07MB/s]"
      }
     },
     "a123680122a944eabcd8acd97fbec793": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_38f20ba38aba4847ae77a6ac2d64503e",
       "placeholder": "​",
       "style": "IPY_MODEL_c7f8d91f3ba84366901e26250d3576ba",
       "value": "Downloading (…)olve/main/merges.txt: 100%"
      }
     },
     "a3c8e58e02d0412a8dc3e612e9dc3aa1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a3ed720903cc462bbbcdb5f22f9a649c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_34f84742e2504e7c95752599c346859f",
       "placeholder": "​",
       "style": "IPY_MODEL_dcd0470ad7804a638b889a20323c3456",
       "value": " 1/1 [00:00&lt;00:00, 66.76it/s]"
      }
     },
     "a46b2a0819d74ba390a68fc5027faa54": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "aa8a4802d77b4a598b1f3eb07070cf99": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_56a016ec42b5449ba91d8d7e3953a2a6",
       "placeholder": "​",
       "style": "IPY_MODEL_42008ceaf47f42d88d5c42e0f2dfe0d3",
       "value": "Downloading (…)okenizer_config.json: 100%"
      }
     },
     "ab89399ed40d47c291750cfe40ae783c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_dae615fc97e74576979eb3d17a2756d9",
       "max": 200.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_e7519d557d204ef3a58fc78cb2c8fe50",
       "value": 200.0
      }
     },
     "b2866ecaa27d477b9f4ac808dee3c8c0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b4bd78c463c64c578a47e694568be5cd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b7aa97505419402b8e4bb004d03b25bd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ba03dc5beb6c49f48062324d04a50ece": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bb809c5eff8a4e73a7cc1f6ef57b9865": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a46b2a0819d74ba390a68fc5027faa54",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_85e65e03b6b44d7a8ee04bb6fee3bfbf",
       "value": 1.0
      }
     },
     "bdfeb000777a494bb8f3ca56a643b88c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "c03a4b842d4e4d16848c38d8351b51a4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_684582618bf64dd69ca566ce365f87b3",
       "max": 1.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d6b25eb2a5344121b0368c7bcee668fb",
       "value": 1.0
      }
     },
     "c25d1a730c0a4bc6a0f0af86d95a3cdd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_920982e62da44f799b7fc6daf813ebaf",
       "placeholder": "​",
       "style": "IPY_MODEL_8461fe7c0381488593e2b990d0cb4cc1",
       "value": "100%"
      }
     },
     "c2c16d89ab774f44888516787fd81201": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e134da70a937426e99b6ff8c253fd999",
       "placeholder": "​",
       "style": "IPY_MODEL_4a973c2b1b72498f879f93511578494f",
       "value": "100%"
      }
     },
     "c479ea86880c480fac79eb26b7bb376e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c609578d923742bf99c74029251db9cd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c63ee1de36fa4b64a2e234cf20c14455": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "c736437bb8214fd9934da4896997653e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1b5febbf5c2c43fd93de5a0a5b9a193b",
       "placeholder": "​",
       "style": "IPY_MODEL_931632e5b2a54c439d7314cc4fe0b8bf",
       "value": " 200/200 [03:01&lt;00:00,  1.08s/it]"
      }
     },
     "c75bd6d7e6fb435b8d7303c280e10185": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4536b2b149cb4f319e6d4beac14343b8",
       "placeholder": "​",
       "style": "IPY_MODEL_0f1f7a8bfc1a4b24bb4e1bfdfc53346e",
       "value": " 1/1 [00:00&lt;00:00, 56.41it/s]"
      }
     },
     "c7f8d91f3ba84366901e26250d3576ba": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "cbbf2f4309434192b4dec22e5cfd0a2b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_82cca1149aca4d7493773bd632c5a81d",
       "max": 90.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_f702e24ab41d4a0094858b05e857a6ec",
       "value": 90.0
      }
     },
     "cbf8631d8c2340cd89cec73c845568e2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "cce73ae2cdf24ad5a4f0cc933b1681c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d63b253c60f1467791a0f6ab1a7353d6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fc65b9679f1c4c6c9056503e47e91cfc",
       "max": 1455.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_95f92684e9b5414aabf27ee6e47af5cd",
       "value": 1455.0
      }
     },
     "d6b25eb2a5344121b0368c7bcee668fb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "d8433b0a7b794d9ba9d7eaa1feed5653": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ea3afeb12fa846d0901905adad647aa8",
       "max": 10672390984.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_c63ee1de36fa4b64a2e234cf20c14455",
       "value": 10672390984.0
      }
     },
     "d851f15680ab4f5482cfcb6e7e762d31": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c25d1a730c0a4bc6a0f0af86d95a3cdd",
        "IPY_MODEL_55b9e9be1f78445facbc1ac028e6a6d8",
        "IPY_MODEL_4be62dbcff2b4e48956775284a743d7b"
       ],
       "layout": "IPY_MODEL_88d056b2043644c0b06b345eb23d7665"
      }
     },
     "dae615fc97e74576979eb3d17a2756d9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dbb0b79d250244b481f4b9d3f48122a2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_a123680122a944eabcd8acd97fbec793",
        "IPY_MODEL_3f19342be98848e0889a1f9ad205ba02",
        "IPY_MODEL_810d8c61866a4422906e88f5feb11fde"
       ],
       "layout": "IPY_MODEL_efdfada142054986b13dad0df1406121"
      }
     },
     "dc634211edb44ffaa56266090a98f742": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dcd0470ad7804a638b889a20323c3456": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "df807131752f4a6b905bdecbf9922f97": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e134da70a937426e99b6ff8c253fd999": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1a2748241ca44f99dc7bbdf670a44f8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e3bb42326dd2444abec7b293fc0e1003": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_aa8a4802d77b4a598b1f3eb07070cf99",
        "IPY_MODEL_652962c0a0404571b3e77019becaa951",
        "IPY_MODEL_910e16a1c8114b7489415836277e5edd"
       ],
       "layout": "IPY_MODEL_483e0d4b53d6443ab9157f1577104a01"
      }
     },
     "e6ff1c7cba1447b9ae5510a3cd0584d6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e7519d557d204ef3a58fc78cb2c8fe50": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "e8a15dc4b6924a8f82c1b58d003a1be7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e1a2748241ca44f99dc7bbdf670a44f8",
       "placeholder": "​",
       "style": "IPY_MODEL_cbf8631d8c2340cd89cec73c845568e2",
       "value": "Downloading (…)olve/main/vocab.json: 100%"
      }
     },
     "ea3afeb12fa846d0901905adad647aa8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ea3ed888dfc04aa0abe6efa8ded27b13": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ba03dc5beb6c49f48062324d04a50ece",
       "placeholder": "​",
       "style": "IPY_MODEL_44e276891b0648f3bf9885d6865fe368",
       "value": " 10.7G/10.7G [01:07&lt;00:00, 170MB/s]"
      }
     },
     "ed83c83fe8254075869f67ae1e888c1d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "efa20add3c1e4d3aaf0ebafdbcac1e73": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "efdfada142054986b13dad0df1406121": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f4df5e05562c47f8834f56417d2ca648": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_7aaec1a022ea498288ba9388d6d0a7cd",
        "IPY_MODEL_c03a4b842d4e4d16848c38d8351b51a4",
        "IPY_MODEL_02f486b763c74748bf9b370911f79165"
       ],
       "layout": "IPY_MODEL_39098750902448cf86250416f5906829"
      }
     },
     "f6d42ddaffc545d39d6074c844e0eba2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f702e24ab41d4a0094858b05e857a6ec": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fc45d24fee47408bb8d38e8abcf2003a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "fc65b9679f1c4c6c9056503e47e91cfc": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ff06c65b4e2141b4a013f9b09b8e0ba3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ff1791d260e848f791297344ab6aaa8d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ffbd928dfb1742be850baacbea258bca": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}