benchmark/draw.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d0cd6522",
   "metadata": {
    "papermill": {
     "duration": 0.006038,
     "end_time": "2024-01-05T15:08:17.835920",
     "exception": false,
     "start_time": "2024-01-05T15:08:17.829882",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Draw Accuracy Benchmarks\n",
    "This is a reproducible benchmark that checks the accuracy of [openskill.py](https://openskill.me/) against a small dataset of chess matches.\n",
    "\n",
    "Let's start by importing the necessary libraries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2ae4cf8b",
   "metadata": {
    "_kg_hide-output": false,
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:17.848615Z",
     "iopub.status.busy": "2024-01-05T15:08:17.848191Z",
     "iopub.status.idle": "2024-01-05T15:08:31.297400Z",
     "shell.execute_reply": "2024-01-05T15:08:31.296096Z"
    },
    "papermill": {
     "duration": 13.458288,
     "end_time": "2024-01-05T15:08:31.299672",
     "exception": false,
     "start_time": "2024-01-05T15:08:17.841384",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (2.0.3)\r\n",
      "Collecting rbo\r\n",
      "  Downloading rbo-0.1.3-py3-none-any.whl (7.8 kB)\r\n",
      "Requirement already satisfied: rich in /opt/conda/lib/python3.10/site-packages (13.5.2)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (1.24.3)\r\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (1.2.2)\r\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (4.66.1)\r\n",
      "Requirement already satisfied: pooch in /opt/conda/lib/python3.10/site-packages (1.8.0)\r\n",
      "Collecting openskill\r\n",
      "  Obtaining dependency information for openskill from https://files.pythonhosted.org/packages/11/af/083022f27cda2608c1ee8b32e759d813b056e1540269b0ae0bbc86226fcb/openskill-5.1.0-py3-none-any.whl.metadata\r\n",
      "  Downloading openskill-5.1.0-py3-none-any.whl.metadata (5.6 kB)\r\n",
      "Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.10/site-packages (7.7.1)\r\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas) (2023.3)\r\n",
      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas) (2023.3)\r\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.10/site-packages (from rich) (3.0.0)\r\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.10/site-packages (from rich) (2.16.1)\r\n",
      "Requirement already satisfied: scipy>=1.3.2 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (1.11.4)\r\n",
      "Requirement already satisfied: joblib>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (1.3.2)\r\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn) (3.2.0)\r\n",
      "Requirement already satisfied: platformdirs>=2.5.0 in /opt/conda/lib/python3.10/site-packages (from pooch) (4.1.0)\r\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from pooch) (21.3)\r\n",
      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from pooch) (2.31.0)\r\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (6.25.1)\r\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (0.2.0)\r\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (5.9.0)\r\n",
      "Requirement already satisfied: widgetsnbextension~=3.6.0 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (3.6.6)\r\n",
      "Requirement already satisfied: ipython>=4.0.0 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (8.14.0)\r\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from ipywidgets) (3.0.8)\r\n",
      "Requirement already satisfied: comm>=0.1.1 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.4)\r\n",
      "Requirement already satisfied: debugpy>=1.6.5 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.6.7.post1)\r\n",
      "Requirement already satisfied: jupyter-client>=6.1.12 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (7.4.9)\r\n",
      "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.3.1)\r\n",
      "Requirement already satisfied: matplotlib-inline>=0.1 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (0.1.6)\r\n",
      "Requirement already satisfied: nest-asyncio in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (1.5.6)\r\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (5.9.3)\r\n",
      "Requirement already satisfied: pyzmq>=20 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (24.0.1)\r\n",
      "Requirement already satisfied: tornado>=6.1 in /opt/conda/lib/python3.10/site-packages (from ipykernel>=4.5.1->ipywidgets) (6.3.3)\r\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (0.2.0)\r\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (5.1.1)\r\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (0.19.0)\r\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (0.7.5)\r\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (3.0.39)\r\n",
      "Requirement already satisfied: stack-data in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (0.6.2)\r\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.10/site-packages (from ipython>=4.0.0->ipywidgets) (4.8.0)\r\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich) (0.1.2)\r\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->pooch) (3.0.9)\r\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas) (1.16.0)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->pooch) (3.2.0)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->pooch) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->pooch) (1.26.15)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->pooch) (2023.11.17)\r\n",
      "Requirement already satisfied: notebook>=4.4.1 in /opt/conda/lib/python3.10/site-packages (from widgetsnbextension~=3.6.0->ipywidgets) (6.5.5)\r\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/conda/lib/python3.10/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets) (0.8.3)\r\n",
      "Requirement already satisfied: entrypoints in /opt/conda/lib/python3.10/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets) (0.4)\r\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.1.2)\r\n",
      "Requirement already satisfied: argon2-cffi in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.3.0)\r\n",
      "Requirement already satisfied: nbformat in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (5.9.2)\r\n",
      "Requirement already satisfied: nbconvert>=5 in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.4.5)\r\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.8.2)\r\n",
      "Requirement already satisfied: terminado>=0.8.3 in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.17.1)\r\n",
      "Requirement already satisfied: prometheus-client in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.17.1)\r\n",
      "Requirement already satisfied: nbclassic>=0.4.7 in /opt/conda/lib/python3.10/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.0.0)\r\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.10/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets) (0.7.0)\r\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.10/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=4.0.0->ipywidgets) (0.2.6)\r\n",
      "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.10/site-packages (from stack-data->ipython>=4.0.0->ipywidgets) (1.2.0)\r\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.10/site-packages (from stack-data->ipython>=4.0.0->ipywidgets) (2.2.1)\r\n",
      "Requirement already satisfied: pure-eval in /opt/conda/lib/python3.10/site-packages (from stack-data->ipython>=4.0.0->ipywidgets) (0.2.2)\r\n",
      "Requirement already satisfied: jupyter-server>=1.8 in /opt/conda/lib/python3.10/site-packages (from nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.12.1)\r\n",
      "Requirement already satisfied: notebook-shim>=0.2.3 in /opt/conda/lib/python3.10/site-packages (from nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.2.3)\r\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.8.4)\r\n",
      "Requirement already satisfied: jupyterlab-pygments in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.2.2)\r\n",
      "Requirement already satisfied: bleach in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.0.0)\r\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.0)\r\n",
      "Requirement already satisfied: testpath in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.6.0)\r\n",
      "Requirement already satisfied: defusedxml in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.7.1)\r\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.12.2)\r\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.13)\r\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.1.3)\r\n",
      "Requirement already satisfied: fastjsonschema in /opt/conda/lib/python3.10/site-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.18.0)\r\n",
      "Requirement already satisfied: jsonschema>=2.6 in /opt/conda/lib/python3.10/site-packages (from nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (4.19.0)\r\n",
      "Requirement already satisfied: argon2-cffi-bindings in /opt/conda/lib/python3.10/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (21.2.0)\r\n",
      "Requirement already satisfied: attrs>=22.2.0 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (23.1.0)\r\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2023.7.1)\r\n",
      "Requirement already satisfied: referencing>=0.28.4 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.30.2)\r\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.9.2)\r\n",
      "Requirement already satisfied: anyio>=3.1.0 in /opt/conda/lib/python3.10/site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (3.7.1)\r\n",
      "Requirement already satisfied: jupyter-events>=0.9.0 in /opt/conda/lib/python3.10/site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.9.0)\r\n",
      "Requirement already satisfied: jupyter-server-terminals in /opt/conda/lib/python3.10/site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.4.4)\r\n",
      "Requirement already satisfied: overrides in /opt/conda/lib/python3.10/site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.5.0)\r\n",
      "Requirement already satisfied: websocket-client in /opt/conda/lib/python3.10/site-packages (from jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.6.2)\r\n",
      "Requirement already satisfied: cffi>=1.0.1 in /opt/conda/lib/python3.10/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.15.1)\r\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.10/site-packages (from beautifulsoup4->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.3.2.post1)\r\n",
      "Requirement already satisfied: webencodings in /opt/conda/lib/python3.10/site-packages (from bleach->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.5.1)\r\n",
      "Requirement already satisfied: sniffio>=1.1 in /opt/conda/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.3.0)\r\n",
      "Requirement already satisfied: exceptiongroup in /opt/conda/lib/python3.10/site-packages (from anyio>=3.1.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.1.3)\r\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.10/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.21)\r\n",
      "Requirement already satisfied: python-json-logger>=2.0.4 in /opt/conda/lib/python3.10/site-packages (from jupyter-events>=0.9.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.0.7)\r\n",
      "Requirement already satisfied: pyyaml>=5.3 in /opt/conda/lib/python3.10/site-packages (from jupyter-events>=0.9.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (6.0.1)\r\n",
      "Requirement already satisfied: rfc3339-validator in /opt/conda/lib/python3.10/site-packages (from jupyter-events>=0.9.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.1.4)\r\n",
      "Requirement already satisfied: rfc3986-validator>=0.1.1 in /opt/conda/lib/python3.10/site-packages (from jupyter-events>=0.9.0->jupyter-server>=1.8->nbclassic>=0.4.7->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (0.1.1)\r\n",
      "Requirement already satisfied: fqdn in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.5.1)\r\n",
      "Requirement already satisfied: isoduration in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (20.11.0)\r\n",
      "Requirement already satisfied: jsonpointer>1.13 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (2.0)\r\n",
      "Requirement already satisfied: uri-template in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.3.0)\r\n",
      "Requirement already satisfied: webcolors>=1.11 in /opt/conda/lib/python3.10/site-packages (from jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.13)\r\n",
      "Requirement already satisfied: arrow>=0.15.0 in /opt/conda/lib/python3.10/site-packages (from isoduration->jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.2.3)\r\n",
      "Downloading openskill-5.1.0-py3-none-any.whl (45 kB)\r\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hInstalling collected packages: rbo, openskill\r\n",
      "Successfully installed openskill-5.1.0 rbo-0.1.3\r\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install pandas rbo rich numpy scikit-learn tqdm pooch openskill ipywidgets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abe29730",
   "metadata": {
    "papermill": {
     "duration": 0.007328,
     "end_time": "2024-01-05T15:08:31.314867",
     "exception": false,
     "start_time": "2024-01-05T15:08:31.307539",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Import Libraries\n",
    "We going to import polars to do the data loading and rich to display things in user friendly format.\n",
    "We are also going to import ipywidget for native iteractivity within the browser."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d87d86d7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:31.332892Z",
     "iopub.status.busy": "2024-01-05T15:08:31.332597Z",
     "iopub.status.idle": "2024-01-05T15:08:33.859952Z",
     "shell.execute_reply": "2024-01-05T15:08:33.858347Z"
    },
    "papermill": {
     "duration": 2.538856,
     "end_time": "2024-01-05T15:08:33.862542",
     "exception": false,
     "start_time": "2024-01-05T15:08:31.323686",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "from dataclasses import dataclass\n",
    "from typing import Dict, List\n",
    "\n",
    "import ipywidgets as widgets\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import rich\n",
    "from rbo import rbo\n",
    "from rich.table import Table\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import openskill\n",
    "from openskill.models import (\n",
    "    BradleyTerryFull,\n",
    "    BradleyTerryPart,\n",
    "    PlackettLuce,\n",
    "    ThurstoneMostellerFull,\n",
    "    ThurstoneMostellerPart,\n",
    ")\n",
    "\n",
    "from pooch import DOIDownloader\n",
    "from pathlib import Path\n",
    "from sklearn.model_selection import train_test_split\n",
    "from enum import Enum"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84103827",
   "metadata": {
    "papermill": {
     "duration": 0.010731,
     "end_time": "2024-01-05T15:08:33.881920",
     "exception": false,
     "start_time": "2024-01-05T15:08:33.871189",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Check OpenSkill Version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d91a702",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:33.903974Z",
     "iopub.status.busy": "2024-01-05T15:08:33.903501Z",
     "iopub.status.idle": "2024-01-05T15:08:33.908571Z",
     "shell.execute_reply": "2024-01-05T15:08:33.907757Z"
    },
    "papermill": {
     "duration": 0.017319,
     "end_time": "2024-01-05T15:08:33.910340",
     "exception": false,
     "start_time": "2024-01-05T15:08:33.893021",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.1.0\n"
     ]
    }
   ],
   "source": [
    "print(openskill.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb1faad4",
   "metadata": {
    "papermill": {
     "duration": 0.007479,
     "end_time": "2024-01-05T15:08:33.925570",
     "exception": false,
     "start_time": "2024-01-05T15:08:33.918091",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Download Data\n",
    "The data we need is not available, yet. So let's download it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9f2523ff",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:33.943028Z",
     "iopub.status.busy": "2024-01-05T15:08:33.942454Z",
     "iopub.status.idle": "2024-01-05T15:08:38.420106Z",
     "shell.execute_reply": "2024-01-05T15:08:38.418108Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "papermill": {
     "duration": 4.490162,
     "end_time": "2024-01-05T15:08:38.423548",
     "exception": false,
     "start_time": "2024-01-05T15:08:33.933386",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0.00B [00:00, ?B/s]\n"
     ]
    }
   ],
   "source": [
    "downloader = DOIDownloader(progressbar=True)\n",
    "\n",
    "working_directory = Path.cwd()\n",
    "data_directory = Path(working_directory / Path(\"data\"))\n",
    "data_directory.mkdir(exist_ok=True)\n",
    "downloader(\n",
    "    url=\"doi:10.5281/zenodo.10344773/chess.csv\",\n",
    "    output_file=data_directory / \"chess.csv\",\n",
    "    pooch=None,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b2a2bbf4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:38.442525Z",
     "iopub.status.busy": "2024-01-05T15:08:38.442191Z",
     "iopub.status.idle": "2024-01-05T15:08:38.449146Z",
     "shell.execute_reply": "2024-01-05T15:08:38.447804Z"
    },
    "papermill": {
     "duration": 0.018902,
     "end_time": "2024-01-05T15:08:38.451226",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.432324",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Result(Enum):\n",
    "    WHITE_WINS = 1\n",
    "    BLACK_WINS = 2\n",
    "    STALEMATE = 3\n",
    "\n",
    "\n",
    "@dataclass(slots=True)\n",
    "class Player:\n",
    "    name: str\n",
    "\n",
    "\n",
    "@dataclass(slots=True)\n",
    "class Match:\n",
    "    result: Result\n",
    "    players: Dict[str, Player]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "42e0b8f5",
   "metadata": {
    "papermill": {
     "duration": 0.009069,
     "end_time": "2024-01-05T15:08:38.468653",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.459584",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Select Model\n",
    "OpenSkill comes with 5 models. Let's pick one for this benchmark."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4bdccd0e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:38.487182Z",
     "iopub.status.busy": "2024-01-05T15:08:38.486702Z",
     "iopub.status.idle": "2024-01-05T15:08:38.499144Z",
     "shell.execute_reply": "2024-01-05T15:08:38.497899Z"
    },
    "papermill": {
     "duration": 0.023878,
     "end_time": "2024-01-05T15:08:38.500985",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.477107",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e0c416fe833447fa6e54dd5c574a3fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Select(description='Model:', index=2, options=('BradleyTerryFull', 'BradleyTerryPart', 'PlackettLuce', 'Thurst…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models = [\n",
    "    BradleyTerryFull,\n",
    "    BradleyTerryPart,\n",
    "    PlackettLuce,\n",
    "    ThurstoneMostellerFull,\n",
    "    ThurstoneMostellerPart,\n",
    "]\n",
    "\n",
    "widget = widgets.Select(\n",
    "    options=[m.__name__ for m in models],\n",
    "    value=PlackettLuce.__name__,\n",
    "    # rows=10,\n",
    "    description=\"Model:\",\n",
    "    disabled=False,\n",
    ")\n",
    "display(widget)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da690e73",
   "metadata": {
    "papermill": {
     "duration": 0.009697,
     "end_time": "2024-01-05T15:08:38.519573",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.509876",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Initialize Model\n",
    "Let's call the constructor on the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "406b04fa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:38.539408Z",
     "iopub.status.busy": "2024-01-05T15:08:38.539025Z",
     "iopub.status.idle": "2024-01-05T15:08:38.545172Z",
     "shell.execute_reply": "2024-01-05T15:08:38.543838Z"
    },
    "papermill": {
     "duration": 0.018232,
     "end_time": "2024-01-05T15:08:38.547029",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.528797",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plackett-Luce Model Parameters: \n",
      "\n",
      "mu: 25.0\n",
      "sigma: 8.333333333333334\n",
      "\n"
     ]
    }
   ],
   "source": [
    "widget.close()\n",
    "m = widget.value\n",
    "m = eval(m)\n",
    "model = m()\n",
    "print(str(model))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d9ae790",
   "metadata": {
    "papermill": {
     "duration": 0.008511,
     "end_time": "2024-01-05T15:08:38.564369",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.555858",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Load Data\n",
    "We are going to use the Pandas to load the CSV file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "feab732a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:38.588544Z",
     "iopub.status.busy": "2024-01-05T15:08:38.588125Z",
     "iopub.status.idle": "2024-01-05T15:08:38.692558Z",
     "shell.execute_reply": "2024-01-05T15:08:38.691806Z"
    },
    "papermill": {
     "duration": 0.121938,
     "end_time": "2024-01-05T15:08:38.695563",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.573625",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load Data\n",
    "df = pd.read_csv(data_directory / \"chess.csv\", index_col=0)\n",
    "\n",
    "# Split Data\n",
    "train, test = train_test_split(df, test_size=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d74d2151",
   "metadata": {
    "papermill": {
     "duration": 0.00989,
     "end_time": "2024-01-05T15:08:38.714326",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.704436",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Let's also hold all rating data in a single variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3d14f2a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:38.733892Z",
     "iopub.status.busy": "2024-01-05T15:08:38.733129Z",
     "iopub.status.idle": "2024-01-05T15:08:38.739125Z",
     "shell.execute_reply": "2024-01-05T15:08:38.737427Z"
    },
    "papermill": {
     "duration": 0.018349,
     "end_time": "2024-01-05T15:08:38.741738",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.723389",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Data Container\n",
    "openskill_players = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7717aeab",
   "metadata": {
    "papermill": {
     "duration": 0.008427,
     "end_time": "2024-01-05T15:08:38.759156",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.750729",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Parse Data for Training Set\n",
    "Let's now parse the training data and put it into our dataclasses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0531c59b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:38.780615Z",
     "iopub.status.busy": "2024-01-05T15:08:38.780009Z",
     "iopub.status.idle": "2024-01-05T15:08:42.927678Z",
     "shell.execute_reply": "2024-01-05T15:08:42.925937Z"
    },
    "papermill": {
     "duration": 4.161566,
     "end_time": "2024-01-05T15:08:42.930097",
     "exception": false,
     "start_time": "2024-01-05T15:08:38.768531",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Raw Data from Training Set into Memory\n",
      "Parsing Training Data:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aaaca9a4f4aa4132af5c6733d72d8315",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46815 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsed 46815 Training Matches\n"
     ]
    }
   ],
   "source": [
    "print(\"Loading Raw Data from Training Set into Memory\")\n",
    "\n",
    "# Parse Training Data\n",
    "print(\"Parsing Training Data:\")\n",
    "\n",
    "# Get Unique Matches for Training\n",
    "train_size = len(train)\n",
    "\n",
    "# Create a Progress Bar\n",
    "t = tqdm(total=train_size)\n",
    "\n",
    "# Training Data\n",
    "train_matches: List[Match] = []\n",
    "\n",
    "for match_index, row in train.iterrows():\n",
    "    white_player = Player(name=row[\"white_username\"])\n",
    "    black_player = Player(name=row[\"black_username\"])\n",
    "    players = {row[\"white_username\"]: white_player, row[\"black_username\"]: black_player}\n",
    "\n",
    "    white_result = row[\"white_result\"]\n",
    "    black_result = row[\"black_result\"]\n",
    "\n",
    "    if white_result == \"win\":\n",
    "        match = Match(result=Result.WHITE_WINS, players=players)\n",
    "    elif black_result == \"win\":\n",
    "        match = Match(result=Result.BLACK_WINS, players=players)\n",
    "    else:\n",
    "        match = Match(result=Result.STALEMATE, players=players)\n",
    "\n",
    "    train_matches.append(match)\n",
    "    t.update(1)\n",
    "\n",
    "print(f\"Parsed {len(train_matches)} Training Matches\")\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f228dfa8",
   "metadata": {
    "papermill": {
     "duration": 0.008431,
     "end_time": "2024-01-05T15:08:42.947401",
     "exception": false,
     "start_time": "2024-01-05T15:08:42.938970",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Intialize OpenSkill Ratings for Train Set\n",
    "Next let's use the ``model.rating`` method to create new rating objects for each player in the training set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c81cdafa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:42.967020Z",
     "iopub.status.busy": "2024-01-05T15:08:42.965880Z",
     "iopub.status.idle": "2024-01-05T15:08:47.996925Z",
     "shell.execute_reply": "2024-01-05T15:08:47.995327Z"
    },
    "papermill": {
     "duration": 5.043722,
     "end_time": "2024-01-05T15:08:47.999620",
     "exception": false,
     "start_time": "2024-01-05T15:08:42.955898",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing Players for Training Set:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e50bf69f9c504d0d9be407ae5b43c22c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46815 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rate Training Matches:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51f148d4e5564186b2a6128060bfb2ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46815 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Initialize OpenSkill Players for Training\n",
    "print(\"Initializing Players for Training Set:\")\n",
    "\n",
    "# Create a Progress Bar\n",
    "t = tqdm(total=train_size)\n",
    "\n",
    "for match in train_matches:\n",
    "    for player_name, player in match.players.items():\n",
    "        player_rating = model.rating(name=player_name)\n",
    "        openskill_players[player_name] = player_rating\n",
    "    t.update(1)\n",
    "\n",
    "# Rate OpenSkill Players for Training\n",
    "print(\"Rate Training Matches:\")\n",
    "t = tqdm(total=len(train_matches))\n",
    "\n",
    "for match in train_matches:\n",
    "    player_1, player_2 = match.players.keys()\n",
    "    player_1_rating = openskill_players[player_1]\n",
    "    player_2_rating = openskill_players[player_2]\n",
    "    team_1 = [player_1_rating]\n",
    "    team_2 = [player_2_rating]\n",
    "\n",
    "    if match.result == Result.WHITE_WINS:\n",
    "        ranks = [1, 2]\n",
    "    elif match.result == Result.BLACK_WINS:\n",
    "        ranks = [2, 1]\n",
    "    else:\n",
    "        ranks = [1, 1]\n",
    "\n",
    "    rated_teams = model.rate(teams=[team_1, team_2], ranks=ranks)\n",
    "\n",
    "    for team in rated_teams:\n",
    "        for player in team:\n",
    "            openskill_players[player.name] = player\n",
    "\n",
    "    t.update(1)\n",
    "\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c7797c4",
   "metadata": {
    "papermill": {
     "duration": 0.010681,
     "end_time": "2024-01-05T15:08:48.021920",
     "exception": false,
     "start_time": "2024-01-05T15:08:48.011239",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Clear Memory\n",
    "Let's delete some variables so we can save some memory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5787e3a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:48.042393Z",
     "iopub.status.busy": "2024-01-05T15:08:48.042004Z",
     "iopub.status.idle": "2024-01-05T15:08:48.163074Z",
     "shell.execute_reply": "2024-01-05T15:08:48.162237Z"
    },
    "papermill": {
     "duration": 0.13304,
     "end_time": "2024-01-05T15:08:48.164681",
     "exception": false,
     "start_time": "2024-01-05T15:08:48.031641",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "del train\n",
    "del train_matches\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19b8fe91",
   "metadata": {
    "papermill": {
     "duration": 0.008988,
     "end_time": "2024-01-05T15:08:48.183434",
     "exception": false,
     "start_time": "2024-01-05T15:08:48.174446",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Parsing Test Set\n",
    "Now let's do the same thing for the test set and grab the matches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7473a58a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:48.203716Z",
     "iopub.status.busy": "2024-01-05T15:08:48.203362Z",
     "iopub.status.idle": "2024-01-05T15:08:49.831407Z",
     "shell.execute_reply": "2024-01-05T15:08:49.829571Z"
    },
    "papermill": {
     "duration": 1.64127,
     "end_time": "2024-01-05T15:08:49.833937",
     "exception": false,
     "start_time": "2024-01-05T15:08:48.192667",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Raw Data from Test Set into Memory\n",
      "Parsing Test Data:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f608c416e6bd44af8405089cd36d9926",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20064 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get Unique Matches for Testing\n",
    "print(\"Loading Raw Data from Test Set into Memory\")\n",
    "\n",
    "test_size = len(test)\n",
    "\n",
    "# Parse Data\n",
    "print(\"Parsing Test Data:\")\n",
    "\n",
    "t = tqdm(total=test_size)\n",
    "\n",
    "# Test Data\n",
    "test_matches: List[Match] = []\n",
    "\n",
    "for match_index, row in test.iterrows():\n",
    "    white_player = Player(name=row[\"white_username\"])\n",
    "    black_player = Player(name=row[\"black_username\"])\n",
    "    players = {row[\"white_username\"]: white_player, row[\"black_username\"]: black_player}\n",
    "\n",
    "    white_result = row[\"white_result\"]\n",
    "    black_result = row[\"black_result\"]\n",
    "\n",
    "    if white_result == \"win\":\n",
    "        match = Match(result=Result.WHITE_WINS, players=players)\n",
    "    elif black_result == \"win\":\n",
    "        match = Match(result=Result.BLACK_WINS, players=players)\n",
    "    else:\n",
    "        match = Match(result=Result.STALEMATE, players=players)\n",
    "\n",
    "    test_matches.append(match)\n",
    "    t.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f6239bc",
   "metadata": {
    "papermill": {
     "duration": 0.009128,
     "end_time": "2024-01-05T15:08:49.852880",
     "exception": false,
     "start_time": "2024-01-05T15:08:49.843752",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Predict Test Set\n",
    "Let's use the data we trained on to try and predict the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "14c941a1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:49.937835Z",
     "iopub.status.busy": "2024-01-05T15:08:49.937479Z",
     "iopub.status.idle": "2024-01-05T15:08:50.530882Z",
     "shell.execute_reply": "2024-01-05T15:08:50.529791Z"
    },
    "papermill": {
     "duration": 0.607105,
     "end_time": "2024-01-05T15:08:50.533311",
     "exception": false,
     "start_time": "2024-01-05T15:08:49.926206",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predict Matches in Test Set using OpenSkill:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc754ce1fa6847c49fc854a40322d4f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20064 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Predict OpenSkill Matches\n",
    "print(\"Predict Matches in Test Set using OpenSkill:\")\n",
    "t = tqdm(total=len(test_matches))\n",
    "\n",
    "# Accuracy Trackers\n",
    "openskill_correct_predictions = 0\n",
    "openskill_incorrect_predictions = 0\n",
    "\n",
    "for match in test_matches:\n",
    "    actual_result = match.result\n",
    "    if actual_result == Result.STALEMATE:\n",
    "        draw = True\n",
    "    else:\n",
    "        draw = False\n",
    "\n",
    "    player_1, player_2 = match.players.keys()\n",
    "\n",
    "    if player_1 in openskill_players:\n",
    "        player_1_rating = openskill_players[player_1]\n",
    "    else:\n",
    "        player_1_rating = model.rating(name=player_1)\n",
    "\n",
    "    if player_2 in openskill_players:\n",
    "        player_2_rating = openskill_players[player_2]\n",
    "    else:\n",
    "        player_2_rating = model.rating(name=player_2)\n",
    "\n",
    "    teams = [[player_1_rating], [player_2_rating]]\n",
    "\n",
    "    white_win_probability, black_win_probability = model.predict_win(teams)\n",
    "    draw_probability = model.predict_draw(teams)\n",
    "\n",
    "    if draw_probability > (white_win_probability + black_win_probability):\n",
    "        if draw:\n",
    "            openskill_correct_predictions += 1\n",
    "        else:\n",
    "            openskill_incorrect_predictions += 1\n",
    "    else:\n",
    "        if not draw:\n",
    "            openskill_correct_predictions += 1\n",
    "        else:\n",
    "            openskill_incorrect_predictions += 1\n",
    "\n",
    "    t.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6af83bf9",
   "metadata": {
    "papermill": {
     "duration": 0.009384,
     "end_time": "2024-01-05T15:08:50.552550",
     "exception": false,
     "start_time": "2024-01-05T15:08:50.543166",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Results\n",
    "Let's print out the result and interpret it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7e08d9a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-05T15:08:50.576084Z",
     "iopub.status.busy": "2024-01-05T15:08:50.575316Z",
     "iopub.status.idle": "2024-01-05T15:08:50.635734Z",
     "shell.execute_reply": "2024-01-05T15:08:50.634112Z"
    },
    "papermill": {
     "duration": 0.075077,
     "end_time": "2024-01-05T15:08:50.638492",
     "exception": false,
     "start_time": "2024-01-05T15:08:50.563415",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-style: italic\">               Benchmark Results               </span>\n",
       "┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\">           Information </span>┃<span style=\"font-weight: bold\"> Value               </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\">     Available Matches </span>│<span style=\"color: #800080; text-decoration-color: #800080\"> 20064               </span>│\n",
       "│<span style=\"color: #008080; text-decoration-color: #008080\"> PlackettLuce Accuracy </span>│<span style=\"color: #800080; text-decoration-color: #800080\"> 19385/679 [ 96.62%] </span>│\n",
       "└───────────────────────┴─────────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[3m               Benchmark Results               \u001b[0m\n",
       "┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1m          Information\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mValue              \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩\n",
       "│\u001b[36m \u001b[0m\u001b[36m    Available Matches\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m20064              \u001b[0m\u001b[35m \u001b[0m│\n",
       "│\u001b[36m \u001b[0m\u001b[36mPlackettLuce Accuracy\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m19385/679 [ 96.62%]\u001b[0m\u001b[35m \u001b[0m│\n",
       "└───────────────────────┴─────────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "table = Table(title=\"Benchmark Results\")\n",
    "table.add_column(\"Information\", justify=\"right\", style=\"cyan\", no_wrap=True)\n",
    "table.add_column(\"Value\", style=\"magenta\")\n",
    "\n",
    "table.add_row(\"Available Matches\", f\"{len(test_matches)}\")\n",
    "\n",
    "openskill_accuracy = round(\n",
    "    (\n",
    "        openskill_correct_predictions\n",
    "        / (openskill_incorrect_predictions + openskill_correct_predictions)\n",
    "    )\n",
    "    * 100,\n",
    "    2,\n",
    ")\n",
    "\n",
    "table.add_row(\n",
    "    f\"{model.__class__.__name__} Accuracy\",\n",
    "    f\"{openskill_correct_predictions}/\"\n",
    "    f\"{openskill_incorrect_predictions} \"\n",
    "    f\"[{openskill_accuracy: .2f}%]\",\n",
    ")\n",
    "rich.print(table)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [],
   "dockerImageVersionId": 30615,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 36.594726,
   "end_time": "2024-01-05T15:08:51.373029",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-01-05T15:08:14.778303",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "034f60764d414065a6700b84f0573687": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "0e0c416fe833447fa6e54dd5c574a3fe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "SelectModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "SelectModel",
       "_options_labels": [
        "BradleyTerryFull",
        "BradleyTerryPart",
        "PlackettLuce",
        "ThurstoneMostellerFull",
        "ThurstoneMostellerPart"
       ],
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "SelectView",
       "description": "Model:",
       "description_tooltip": null,
       "disabled": false,
       "index": 2,
       "layout": "IPY_MODEL_4cbccfbca2d54f75be5c1eca664e96eb",
       "rows": 5,
       "style": "IPY_MODEL_b17a867bc6ba4d40bdf65b5dc0f65c35"
      }
     },
     "0f0e5f17a54e4f8caa77c24a6cc4c828": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3e67c2f533cc40b48554c5ad19099ae2",
       "placeholder": "​",
       "style": "IPY_MODEL_db035ec20d38489c8e165c3e8e9e0f4b",
       "value": " 87%"
      }
     },
     "172be0c19e14451b9a89f7746fe70d27": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "19425e84ca6f42889c129a17b4295de6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1ae480dd4b3848ac923af6e09738f6ca": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "238e7b588ce44b06bdeaee7b2440cf8d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2ad3d5ff053048639b0e8fda97a6ea0e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f9eb9b1a80c84501aea060d75769ef07",
       "max": 20064.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_37e0d15f6b5a45ab8f63a35456d3d09e",
       "value": 17413.0
      }
     },
     "2de420e4e8674afb81a57a9e1e63ab47": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_940d3945f33444cd89e4599731133741",
       "placeholder": "​",
       "style": "IPY_MODEL_172be0c19e14451b9a89f7746fe70d27",
       "value": " 46815/46815 [00:05&lt;00:00, 69444.28it/s]"
      }
     },
     "3324db69a4db4f469c5ed0632209f736": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "37e0d15f6b5a45ab8f63a35456d3d09e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "3e67c2f533cc40b48554c5ad19099ae2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "496894d12e344ce9b0fa41f3cc7bdb0a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4a20f554d2f144cc963c79e9099c83ae": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "4cbccfbca2d54f75be5c1eca664e96eb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4d87fe4dcc8d4f21a6797dfc629d3852": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "51f148d4e5564186b2a6128060bfb2ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_9c950fbf919944259a0a44f2595febb2",
        "IPY_MODEL_b9471c4da6b14c98808c949a05e1fb36",
        "IPY_MODEL_d4e117d8f23a47389d8b5da6c8b3d528"
       ],
       "layout": "IPY_MODEL_238e7b588ce44b06bdeaee7b2440cf8d"
      }
     },
     "55e332bdcbbc41608ccbee860b039671": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "59057b5196b34fa7b0f3233bdd7f940c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c87e2469f6014b62bee4f9038689ef69",
       "placeholder": "​",
       "style": "IPY_MODEL_a5441fbc21ca4fb19c0797aa1c0b4b14",
       "value": "100%"
      }
     },
     "5ca34deb6a25475fa96629fdeee9aabb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7284ae570dab45f1b9b2839d00b33217": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "78749e6f7231449c86f2e1cb0a91f7e0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_55e332bdcbbc41608ccbee860b039671",
       "max": 46815.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_92c38d4ea7eb42c2a5db3a12038bb35c",
       "value": 46815.0
      }
     },
     "7c2d4fa6c01442cc944e4e9529bba559": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "836f9e463df84aeaacc33675473c1086": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e3453775e5ea412193efd76e3e9fea5e",
       "max": 20064.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ae1d7dc4e2274574bda97c6eb2ade02f",
       "value": 20063.0
      }
     },
     "8fe77284396e455aab27e5b0d3f0c48f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "91f5834dbb464ab1a907a2b5859a5bad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "92c38d4ea7eb42c2a5db3a12038bb35c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "92e1e1cd0bf64157bdf4b540aea58b45": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9365b00f34f44493a1a463a16e5da496": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "940d3945f33444cd89e4599731133741": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "94fa432969c0406bb4651ff86ccd8eb3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_3324db69a4db4f469c5ed0632209f736",
       "placeholder": "​",
       "style": "IPY_MODEL_ed2dd4458ede4b30a0a61a8ca8606829",
       "value": "100%"
      }
     },
     "9b58646441384fe086f8ca9e7bc7a729": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "9c950fbf919944259a0a44f2595febb2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_034f60764d414065a6700b84f0573687",
       "placeholder": "​",
       "style": "IPY_MODEL_4d87fe4dcc8d4f21a6797dfc629d3852",
       "value": " 98%"
      }
     },
     "9fab19c98ec6443a8ed79c69875cd9f5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7c2d4fa6c01442cc944e4e9529bba559",
       "placeholder": "​",
       "style": "IPY_MODEL_c8dd325d30ae4b04b489d66ea807086c",
       "value": " 46815/46815 [00:09&lt;00:00, 11642.03it/s]"
      }
     },
     "a5441fbc21ca4fb19c0797aa1c0b4b14": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "aaaca9a4f4aa4132af5c6733d72d8315": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_59057b5196b34fa7b0f3233bdd7f940c",
        "IPY_MODEL_ac85538827ef408bb149c5725622ec62",
        "IPY_MODEL_9fab19c98ec6443a8ed79c69875cd9f5"
       ],
       "layout": "IPY_MODEL_cd09c77295ec49adb51ca37413d38a91"
      }
     },
     "abbfc729e4034847a54c3782d10c6253": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ac85538827ef408bb149c5725622ec62": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7284ae570dab45f1b9b2839d00b33217",
       "max": 46815.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_91f5834dbb464ab1a907a2b5859a5bad",
       "value": 46815.0
      }
     },
     "ae1d7dc4e2274574bda97c6eb2ade02f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "b17a867bc6ba4d40bdf65b5dc0f65c35": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b9471c4da6b14c98808c949a05e1fb36": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_19425e84ca6f42889c129a17b4295de6",
       "max": 46815.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_4a20f554d2f144cc963c79e9099c83ae",
       "value": 46038.0
      }
     },
     "bc754ce1fa6847c49fc854a40322d4f6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_0f0e5f17a54e4f8caa77c24a6cc4c828",
        "IPY_MODEL_2ad3d5ff053048639b0e8fda97a6ea0e",
        "IPY_MODEL_e8c182e671834605bd3d5168b9679be3"
       ],
       "layout": "IPY_MODEL_f9260511282b4c24b5d229b1db84f23f"
      }
     },
     "bdec7b9fafcb4c58beb782ff9cdd8eac": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bf0c8f8816c24297918bd08096f90520": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bdec7b9fafcb4c58beb782ff9cdd8eac",
       "placeholder": "​",
       "style": "IPY_MODEL_abbfc729e4034847a54c3782d10c6253",
       "value": " 20063/20064 [00:01&lt;00:00, 12510.17it/s]"
      }
     },
     "c87e2469f6014b62bee4f9038689ef69": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c8dd325d30ae4b04b489d66ea807086c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "cd09c77295ec49adb51ca37413d38a91": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d1d541115cfe4cb8b9ba143a2e8bb65b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_5ca34deb6a25475fa96629fdeee9aabb",
       "placeholder": "​",
       "style": "IPY_MODEL_9b58646441384fe086f8ca9e7bc7a729",
       "value": "100%"
      }
     },
     "d4e117d8f23a47389d8b5da6c8b3d528": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_92e1e1cd0bf64157bdf4b540aea58b45",
       "placeholder": "​",
       "style": "IPY_MODEL_1ae480dd4b3848ac923af6e09738f6ca",
       "value": " 46038/46815 [00:04&lt;00:00, 11049.21it/s]"
      }
     },
     "d80d332787554a62b73494eb788fa9d0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "db035ec20d38489c8e165c3e8e9e0f4b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e3453775e5ea412193efd76e3e9fea5e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e50bf69f9c504d0d9be407ae5b43c22c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_d1d541115cfe4cb8b9ba143a2e8bb65b",
        "IPY_MODEL_78749e6f7231449c86f2e1cb0a91f7e0",
        "IPY_MODEL_2de420e4e8674afb81a57a9e1e63ab47"
       ],
       "layout": "IPY_MODEL_496894d12e344ce9b0fa41f3cc7bdb0a"
      }
     },
     "e8c182e671834605bd3d5168b9679be3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8fe77284396e455aab27e5b0d3f0c48f",
       "placeholder": "​",
       "style": "IPY_MODEL_9365b00f34f44493a1a463a16e5da496",
       "value": " 17413/20064 [00:00&lt;00:00, 34359.82it/s]"
      }
     },
     "ed2dd4458ede4b30a0a61a8ca8606829": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "f608c416e6bd44af8405089cd36d9926": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_94fa432969c0406bb4651ff86ccd8eb3",
        "IPY_MODEL_836f9e463df84aeaacc33675473c1086",
        "IPY_MODEL_bf0c8f8816c24297918bd08096f90520"
       ],
       "layout": "IPY_MODEL_d80d332787554a62b73494eb788fa9d0"
      }
     },
     "f9260511282b4c24b5d229b1db84f23f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f9eb9b1a80c84501aea060d75769ef07": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}