german_bert_ner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9_Igu3_P7fxV",
   "metadata": {
    "id": "9_Igu3_P7fxV"
   },
   "source": [
    "## Fine-tune **[`German BERT`](https://www.deepset.ai/german-bert)** on German Legal Data\n",
    "\n",
    "---\n",
    "**Important**: To successfully execute this notebook, make sure you have access to a GPU.\n",
    "\n",
    "**Dataset**: https://github.com/elenanereiss/Legal-Entity-Recognition/tree/master/data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "KsjC8sN-qUa4",
   "metadata": {
    "id": "KsjC8sN-qUa4"
   },
   "source": [
    "### Install and import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60uDEtc10LTN",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "60uDEtc10LTN",
    "outputId": "44f043f1-c4af-426c-87ac-04cf586fb5db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras in /usr/local/lib/python3.7/dist-packages (2.4.3)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from keras) (3.1.0)\n",
      "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.7/dist-packages (from keras) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.7/dist-packages (from keras) (1.19.5)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from keras) (3.13)\n",
      "Requirement already satisfied: cached-property; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from h5py->keras) (1.5.2)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (0.22.2.post1)\n",
      "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn) (1.19.5)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn) (1.4.1)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn) (1.0.1)\n",
      "Collecting transformers\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/1a/41c644c963249fd7f3836d926afa1e3f1cc234a1c40d80c5f03ad8f6f1b2/transformers-4.8.2-py3-none-any.whl (2.5MB)\n",
      "\u001b[K     |████████████████████████████████| 2.5MB 9.3MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
      "Collecting huggingface-hub==0.0.12\n",
      "  Downloading https://files.pythonhosted.org/packages/2f/ee/97e253668fda9b17e968b3f97b2f8e53aa0127e8807d24a547687423fe0b/huggingface_hub-0.0.12-py3-none-any.whl\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
      "Collecting tokenizers<0.11,>=0.10.1\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
      "\u001b[K     |████████████████████████████████| 3.3MB 58.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
      "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.0)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
      "\u001b[K     |████████████████████████████████| 901kB 40.6MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
      "Installing collected packages: huggingface-hub, tokenizers, sacremoses, transformers\n",
      "Successfully installed huggingface-hub-0.0.12 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.8.2\n",
      "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.9.0+cu102)\n",
      "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.10.0+cu102)\n",
      "Collecting torchaudio\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a8/20/eab40caad8f4b97f5e91a5de8ba5ec29115e08fa4c9a808725490b7b4844/torchaudio-0.9.0-cp37-cp37m-manylinux1_x86_64.whl (1.9MB)\n",
      "\u001b[K     |████████████████████████████████| 1.9MB 7.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
      "Requirement already satisfied: pillow>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.19.5)\n",
      "Installing collected packages: torchaudio\n",
      "Successfully installed torchaudio-0.9.0\n"
     ]
    }
   ],
   "source": [
    "!pip install keras\n",
    "!pip install scikit-learn\n",
    "!pip install transformers\n",
    "!pip install torch torchvision torchaudio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6339b2cd-dc1d-4504-bb3d-11f2375cc674",
   "metadata": {
    "id": "6339b2cd-dc1d-4504-bb3d-11f2375cc674"
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "import transformers\n",
    "from transformers import BertTokenizer, BertConfig\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "from transformers import BertForTokenClassification, AdamW\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import multilabel_confusion_matrix\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "iJlsQfj3oLUW",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "iJlsQfj3oLUW",
    "outputId": "89f9ae87-ce83-4af9-de48-92add8100838"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'1.9.0+cu102'"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "921YJhIm0Tzp",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "921YJhIm0Tzp",
    "outputId": "2c169ccf-c30b-437f-9d7c-ce791b89bd5f"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'Tesla T4'"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda')\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.get_device_name(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d288dfb7-d29e-4ea2-b26d-4106ae2e36bf",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "d288dfb7-d29e-4ea2-b26d-4106ae2e36bf",
    "outputId": "a95c5a95-76bd-43e0-aa68-71622afd7e05"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'4.8.2'"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformers.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "HjypuaJMrB6e",
   "metadata": {
    "id": "HjypuaJMrB6e"
   },
   "source": [
    "### Set-up data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "090065c1-1595-4092-8448-d2bd38a41a1c",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 201
    },
    "id": "090065c1-1595-4092-8448-d2bd38a41a1c",
    "outputId": "bdcd4176-cfd8-42dc-ef7e-e8c17d5c570d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_number</th>\n",
       "      <th>word</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2157043</th>\n",
       "      <td>66722</td>\n",
       "      <td>135</td>\n",
       "      <td>I-GS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2157044</th>\n",
       "      <td>66722</td>\n",
       "      <td>Abs.</td>\n",
       "      <td>I-GS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2157045</th>\n",
       "      <td>66722</td>\n",
       "      <td>1</td>\n",
       "      <td>I-GS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2157046</th>\n",
       "      <td>66722</td>\n",
       "      <td>FGO</td>\n",
       "      <td>I-GS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2157047</th>\n",
       "      <td>66722</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sentence_number  word   tag\n",
       "2157043            66722   135  I-GS\n",
       "2157044            66722  Abs.  I-GS\n",
       "2157045            66722     1  I-GS\n",
       "2157046            66722   FGO  I-GS\n",
       "2157047            66722     .     O"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('court_data.csv', sep='|', quoting=csv.QUOTE_NONE).fillna(method='ffill')\n",
    "data.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lEJs_lEQrxco",
   "metadata": {
    "id": "lEJs_lEQrxco"
   },
   "source": [
    "### Set-up data iterator"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7BW8dHEZ4SF-",
   "metadata": {
    "id": "7BW8dHEZ4SF-"
   },
   "source": [
    "The class **`GetSentence`** returns a list of tokenized sentence and its corresponding labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d5659e33-701f-45fb-8a7b-34952c5126a2",
   "metadata": {
    "id": "d5659e33-701f-45fb-8a7b-34952c5126a2"
   },
   "outputs": [],
   "source": [
    "class GetSentence(object):\n",
    "    def __init__(self, data):\n",
    "        self.n_sent = 1\n",
    "        self.data = data\n",
    "        self.empty = False\n",
    "        agg = lambda s: [(w, t) for w, t in zip(s['word'].values.tolist(), s['tag'].values.tolist())]\n",
    "        self.grouped = self.data.groupby('sentence_number').apply(agg)\n",
    "        self.sentences = [s for s in self.grouped]\n",
    "        \n",
    "    def get_next(self):\n",
    "        try:\n",
    "            s = self.grouped['{}'.format(self.n_sent)]\n",
    "            self.n_sent += 1\n",
    "            return s\n",
    "        except:\n",
    "            return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4e4903ca-a764-4707-b0f9-9e7425c67c2f",
   "metadata": {
    "id": "4e4903ca-a764-4707-b0f9-9e7425c67c2f"
   },
   "outputs": [],
   "source": [
    "getter = GetSentence(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "816603d0-cc8d-4b72-a995-44a488056659",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "816603d0-cc8d-4b72-a995-44a488056659",
    "outputId": "1e312398-870d-4e3a-eee2-aea271dfdee2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['In',\n",
       " 'der',\n",
       " 'Beschwerdesache',\n",
       " 'betreffend',\n",
       " 'die',\n",
       " 'Marke',\n",
       " '30',\n",
       " '2012',\n",
       " '044',\n",
       " '496',\n",
       " 'hat',\n",
       " 'der',\n",
       " '25.',\n",
       " 'Senat',\n",
       " '(',\n",
       " 'Marken-Beschwerdesenat',\n",
       " ')',\n",
       " 'des',\n",
       " 'Bundespatentgerichts',\n",
       " 'am',\n",
       " '18.',\n",
       " 'Oktober',\n",
       " '2017',\n",
       " 'unter',\n",
       " 'Mitwirkung',\n",
       " 'des',\n",
       " 'Vorsitzenden',\n",
       " 'Richters',\n",
       " 'Knoll',\n",
       " ',',\n",
       " 'der',\n",
       " 'Richterin',\n",
       " 'Kriener',\n",
       " 'und',\n",
       " 'des',\n",
       " 'Richters',\n",
       " 'Dr.',\n",
       " 'Nielsen',\n",
       " 'beschlossen',\n",
       " ':']"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = [[word[0] for word in sentence] for sentence in getter.sentences]\n",
    "sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d22fdc36-f2c5-498c-bf52-add1dee8fe55",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d22fdc36-f2c5-498c-bf52-add1dee8fe55",
    "outputId": "6bde0083-e04d-43c5-9d77-9c956738053e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-GRT', 'I-GRT', 'I-GRT', 'I-GRT', 'I-GRT', 'I-GRT', 'I-GRT', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-RR', 'O', 'O', 'O', 'B-RR', 'O', 'O', 'O', 'O', 'B-RR', 'O', 'O']\n"
     ]
    }
   ],
   "source": [
    "labels = [[s[1] for s in sentence] for sentence in getter.sentences]\n",
    "print(labels[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "TbeLFeTOsXDj",
   "metadata": {
    "id": "TbeLFeTOsXDj"
   },
   "source": [
    "### Set of unique tags and its indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c5b1d924-4e5d-4a4d-ae95-dd06a27ccc3b",
   "metadata": {
    "id": "c5b1d924-4e5d-4a4d-ae95-dd06a27ccc3b"
   },
   "outputs": [],
   "source": [
    "tag_values = list(set(data['tag'].values))\n",
    "tag_values.append('PAD')\n",
    "tag2idx = {t: i for i, t in enumerate(tag_values)}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "S645qam-sit2",
   "metadata": {
    "id": "S645qam-sit2"
   },
   "source": [
    "Save **`tag_values`** as it will be required for later use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ysygt8pbu4SY",
   "metadata": {
    "id": "ysygt8pbu4SY"
   },
   "outputs": [],
   "source": [
    "t_values = open(\"tag_values.pkl\", \"wb\")\n",
    "pickle.dump(tag_values, t_values)\n",
    "t_values.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pQ1g-VO-s-Lf",
   "metadata": {
    "id": "pQ1g-VO-s-Lf"
   },
   "source": [
    "### Set-up BERT tokenizer from pre-trained **`bert-base-german-cased`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "885ae799-c5f1-4edb-9020-b0106fa53eb4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 162,
     "referenced_widgets": [
      "ea67678edff04c4bae2d16685019e388",
      "d8063e3b2ce74d88a6ee76e2eb374c08",
      "cf23bbaa269c4140a7c4b2f03b6fe129",
      "91f92d528872463faf922b67d4f04843",
      "b808334ea7074334ac496201fe7983b8",
      "db7404baf0df442ebd3d552299fc2c40",
      "0da9cfa444d9431f9e4af04ecda9eaf7",
      "a49f32a2a95041e8addc4cf61ee285d6",
      "d2c9b4da73e445c8a2a7cef1d1e0a6d6",
      "0c37edd5fa00484485f0b891275cc0cd",
      "17a4860b5c334a9a9816f16c9e540d47",
      "05597062f4a74a0c89ffe6371cbfc04b",
      "0ea4e6c53c504a95bde6cf2aff60ce24",
      "7cd17bb918e94717a051ed6ab416cd51",
      "2c3bbdb526e148c2ac9f327aae1c353d",
      "42d23d861ffb44baa2941e3640f01734",
      "40f855988bd045a18b54bd735288812d",
      "1fc085f7317843eb871b60557b16f27a",
      "8134a304285b471cb8838397949d2fcb",
      "d77a82b18daf4528a2a033cfb1dd2155",
      "0a0ac7e623834325830806e9a780c3b0",
      "ca6504116ee849d098c7404eab27ba85",
      "e0ac7da8f2014ca79cb65e5bc4dd6436",
      "0ab85e4295174870a5541fe8fb0f6439"
     ]
    },
    "id": "885ae799-c5f1-4edb-9020-b0106fa53eb4",
    "outputId": "9ac21cdb-eddb-4a11-96b2-62de2c40bd48"
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-german-cased', do_lower_case=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21XtumKDtKXp",
   "metadata": {
    "id": "21XtumKDtKXp"
   },
   "source": [
    "As with `tag_values`, we will also require **`tokenizer`** for later use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "gZExDq5xYBFV",
   "metadata": {
    "id": "gZExDq5xYBFV"
   },
   "outputs": [],
   "source": [
    "save_tokenizer = open(\"tokenizer.pkl\", \"wb\")\n",
    "pickle.dump(tokenizer, save_tokenizer)\n",
    "save_tokenizer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "_Zfk56-zuOiw",
   "metadata": {
    "id": "_Zfk56-zuOiw"
   },
   "source": [
    "Since BERT uses **WordPiece**, we also have to make our sentences to similar format.\n",
    "\n",
    "The following function accepts **`sentences`** and **`labels`**, and iterates through every single one of them.\n",
    "\n",
    "Our **`tokenizer`** is applied to every single word from each sentence of **`sentences`**. While doing this, we have to make each sub-word from word has the same label."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bb56f8db-617d-42d6-b738-49442b271af2",
   "metadata": {
    "id": "bb56f8db-617d-42d6-b738-49442b271af2"
   },
   "outputs": [],
   "source": [
    "def tokenize_preserve_labels(sentence, text_labels):\n",
    "    tokenized_sentence = []\n",
    "    labels = []\n",
    "    \n",
    "    for word, label in zip(sentence, text_labels):\n",
    "        tokenized_word = tokenizer.tokenize(word)\n",
    "        n_subwords = len(tokenized_word)\n",
    "        tokenized_sentence.extend(tokenized_word)\n",
    "        labels.extend([label] * n_subwords)\n",
    "        \n",
    "    return tokenized_sentence, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d5c72c90-d085-4210-b7b3-615b7ba3a026",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d5c72c90-d085-4210-b7b3-615b7ba3a026",
    "outputId": "6f0c7cd0-3d9b-46c7-fc11-72f533ccee99"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 45s, sys: 419 ms, total: 1min 45s\n",
      "Wall time: 1min 45s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tokenized_texts_labels = [tokenize_preserve_labels(sent, labels) for sent, labels in zip(sentences, labels)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6i8uiY4zvsdN",
   "metadata": {
    "id": "6i8uiY4zvsdN"
   },
   "source": [
    "Extract **tokens** and **labels** from **`tokenized_texts_labels`**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6fcb10a0-3573-4588-8c66-4877c3a4ac75",
   "metadata": {
    "id": "6fcb10a0-3573-4588-8c66-4877c3a4ac75"
   },
   "outputs": [],
   "source": [
    "tokenized_texts = [token_label_pair[0] for token_label_pair in tokenized_texts_labels]\n",
    "labels = [token_label_pair[1] for token_label_pair in tokenized_texts_labels]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "OG7bd0TAwJ3B",
   "metadata": {
    "id": "OG7bd0TAwJ3B"
   },
   "source": [
    "### Apply padding and generate **`attention_mask`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7d7822b5-c6d2-430f-af5e-8b0dde24ce67",
   "metadata": {
    "id": "7d7822b5-c6d2-430f-af5e-8b0dde24ce67"
   },
   "outputs": [],
   "source": [
    "MAX_LEN = 75\n",
    "BATCH_SIZE = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b2ebe348-5d60-4b5b-91e4-1dbe3f990108",
   "metadata": {
    "id": "b2ebe348-5d60-4b5b-91e4-1dbe3f990108"
   },
   "outputs": [],
   "source": [
    "input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts], maxlen=MAX_LEN, dtype='long', value=0.0, truncating='post', padding='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a1a8800f-5260-4f48-b562-64448f1925e1",
   "metadata": {
    "id": "a1a8800f-5260-4f48-b562-64448f1925e1"
   },
   "outputs": [],
   "source": [
    "tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels], maxlen=MAX_LEN, value=tag2idx['PAD'], padding='post', dtype='long', truncating='post')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c8a4196b-d5b9-4422-a304-cc42cd0303c8",
   "metadata": {
    "id": "c8a4196b-d5b9-4422-a304-cc42cd0303c8"
   },
   "outputs": [],
   "source": [
    "attention_mask = [[float(i != 0.0) for i in ii] for ii in input_ids]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "Lh1AVCfH0cRs",
   "metadata": {
    "id": "Lh1AVCfH0cRs"
   },
   "source": [
    "### Prepare training and testing data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "H6JKcK_02Ln-",
   "metadata": {
    "id": "H6JKcK_02Ln-"
   },
   "source": [
    "Split data and attention mask."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "beb85d4d-fc6c-46c1-9528-cf6b510d2dde",
   "metadata": {
    "id": "beb85d4d-fc6c-46c1-9528-cf6b510d2dde"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(input_ids, tags, random_state=42, test_size=0.1)\n",
    "tr_mask, val_mask, _, _ = train_test_split(attention_mask, input_ids, random_state=42, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "033526d0-ce2c-450b-a807-bf9112ff7444",
   "metadata": {
    "id": "033526d0-ce2c-450b-a807-bf9112ff7444"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = torch.tensor(X_train), torch.tensor(X_test), torch.tensor(y_train), torch.tensor(y_test)\n",
    "tr_mask, val_mask = torch.tensor(tr_mask), torch.tensor(val_mask)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "zikhaibD2OId",
   "metadata": {
    "id": "zikhaibD2OId"
   },
   "source": [
    "Create data-loaders."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "94639695-e977-4e7a-820a-0135dfa74c8f",
   "metadata": {
    "id": "94639695-e977-4e7a-820a-0135dfa74c8f"
   },
   "outputs": [],
   "source": [
    "train_data = TensorDataset(X_train, tr_mask, y_train)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
    "\n",
    "valid_data = TensorDataset(X_test, val_mask, y_test)\n",
    "valid_sampler = SequentialSampler(valid_data)\n",
    "valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "nZFHGBn71ndF",
   "metadata": {
    "id": "nZFHGBn71ndF"
   },
   "source": [
    "### Pull and fine-tune **`bert-base-german-cased`** model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da724751-3626-43b3-a979-435040595400",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 216,
     "referenced_widgets": [
      "ead333525fcc4534ab7ce6bb8a828add",
      "3ca198d81a814391b45672b3fbf5cb70",
      "3825314e00644a83af7564114c30d356",
      "54b494a263f6486ab23bdf73cdfd107d",
      "242defaf6f9c42bfa548ec67c27efa3c",
      "ae5551d860794fb1b3fcf5325ec331ad",
      "a076cfbd889c483c9c73722ec21f8ba1",
      "032d826cd1ce4b82bc151ca9e957f6c5",
      "869b0fb93fad468692c4edd48b5d8488",
      "549ea47dba7343ceb80b52b64231906d",
      "240e9994af274a7c806360535c56f977",
      "4929f2c059f948cdb526cce1acd74232",
      "de3803d547a6450d8eeabf7bacab9e34",
      "0079fc67b85841579d235bcc63039aba",
      "372a3a41433b47a3bc271cf0f4e4ee5f",
      "d841d68e45764d85b3e7c29243421230"
     ]
    },
    "id": "da724751-3626-43b3-a979-435040595400",
    "outputId": "c1b4591b-7500-435f-b5ba-1ee0f2dc2abf"
   },
   "outputs": [],
   "source": [
    "model = BertForTokenClassification.from_pretrained('bert-base-german-cased', num_labels=len(tag2idx), output_attentions=False, output_hidden_states=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "PQEHRctF0xmz",
   "metadata": {
    "id": "PQEHRctF0xmz"
   },
   "outputs": [],
   "source": [
    "model.cuda();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "1db826e5-d274-41f0-864e-5d983479b9fe",
   "metadata": {
    "id": "1db826e5-d274-41f0-864e-5d983479b9fe"
   },
   "outputs": [],
   "source": [
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters)\n",
    "    optimizer_grouped_parameters = [{'params': [p for n, p in param_optimizer]}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7ae22876-5c98-4a7e-b548-d8851b25e7e4",
   "metadata": {
    "id": "7ae22876-5c98-4a7e-b548-d8851b25e7e4"
   },
   "outputs": [],
   "source": [
    "optimizer = AdamW(optimizer_grouped_parameters, lr=3e-5, eps=1e-8)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "zuy0Po8W2cI-",
   "metadata": {
    "id": "zuy0Po8W2cI-"
   },
   "source": [
    "### Training and evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c2d8bbff-7dd7-4f76-8e5b-333633ca9eff",
   "metadata": {
    "id": "c2d8bbff-7dd7-4f76-8e5b-333633ca9eff"
   },
   "outputs": [],
   "source": [
    "EPOCHS = 3\n",
    "MAX_GRAD_NORM = 1.0\n",
    "\n",
    "total_steps = len(train_dataloader) * EPOCHS\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9352258c-a388-4f0d-88f7-b1d38404e120",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9352258c-a388-4f0d-88f7-b1d38404e120",
    "outputId": "41d7fbf9-27e6-4e4c-ab73-323a935543e4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Epoch 01 -\n",
      "Average train loss:\t0.08197\n",
      "Validation loss:\t0.02941\n",
      "Validation accuracy:\t0.99195\n",
      "Validation precision:\t0.99195\n",
      "Validation recall:\t0.99195\n",
      "Validation f1-score:\t0.99195\n",
      "\n",
      "- Epoch 02 -\n",
      "Average train loss:\t0.01605\n",
      "Validation loss:\t0.02284\n",
      "Validation accuracy:\t0.99389\n",
      "Validation precision:\t0.99389\n",
      "Validation recall:\t0.99389\n",
      "Validation f1-score:\t0.99389\n",
      "\n",
      "- Epoch 03 -\n",
      "Average train loss:\t0.00752\n",
      "Validation loss:\t0.02323\n",
      "Validation accuracy:\t0.99488\n",
      "Validation precision:\t0.99488\n",
      "Validation recall:\t0.99488\n",
      "Validation f1-score:\t0.99488\n",
      "\n",
      "CPU times: user 45min 54s, sys: 16.1 s, total: 46min 11s\n",
      "Wall time: 46min 17s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "loss_values, validation_loss_values = [], []\n",
    "\n",
    "for e in range(EPOCHS):\n",
    "    print(f'- Epoch 0{e+1} -')\n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        model.zero_grad()\n",
    "        outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
    "        loss = outputs[0]\n",
    "        loss.backward()\n",
    "        total_loss += loss.item()\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=MAX_GRAD_NORM)\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        \n",
    "    avg_train_loss = total_loss / len(train_dataloader)\n",
    "    print('Average train loss:\\t{:.5f}'.format(avg_train_loss))\n",
    "    loss_values.append(avg_train_loss)\n",
    "    \n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    predictions, true_labels = [], []\n",
    "    \n",
    "    for batch in valid_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
    "            \n",
    "        logits = outputs[1].detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        \n",
    "        eval_loss += outputs[0].mean().item()\n",
    "        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        true_labels.extend(label_ids)\n",
    "        \n",
    "    eval_loss = eval_loss / len(valid_dataloader)\n",
    "    validation_loss_values.append(eval_loss)\n",
    "    print('Validation loss:\\t{:.5f}'.format(eval_loss))\n",
    "    \n",
    "    pred_tags = [tag_values[p_i] for p, l in zip(predictions, true_labels) for p_i, l_i in zip(p, l) if tag_values[l_i] != 'PAD']\n",
    "    valid_tags = [tag_values[l_i] for l in true_labels for l_i in l if tag_values[l_i] != 'PAD']\n",
    "\n",
    "    print('Validation accuracy:\\t{:.5f}'.format(accuracy_score(pred_tags, valid_tags)))\n",
    "    print('Validation precision:\\t{:.5f}'.format(precision_score(pred_tags, valid_tags, average='micro')))\n",
    "    print('Validation recall:\\t{:.5f}'.format(recall_score(pred_tags, valid_tags, average='micro')))\n",
    "    print('Validation f1-score:\\t{:.5f}\\n'.format(f1_score(pred_tags, valid_tags, average='micro')))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "LARGmk0j2oZu",
   "metadata": {
    "id": "LARGmk0j2oZu"
   },
   "source": [
    "Calculate confusion matrix to identify **TP**, **TN**, **FP**, and **FN**. This is required to calculate **Micro- precision**, **recall**, and **F1-Score**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "qnT6goDTWV9J",
   "metadata": {
    "id": "qnT6goDTWV9J"
   },
   "outputs": [],
   "source": [
    "tags = list(set(valid_tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "znd6ZEEwAR4N",
   "metadata": {
    "id": "znd6ZEEwAR4N"
   },
   "outputs": [],
   "source": [
    "matrix = multilabel_confusion_matrix(valid_tags, pred_tags, labels=tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "shQoT-pHBHJq",
   "metadata": {
    "id": "shQoT-pHBHJq"
   },
   "outputs": [],
   "source": [
    "tags_eval = {}\n",
    "for t, m in zip(tags, matrix):\n",
    "    tag = t.split('-')[-1]\n",
    "    if tag not in tags_eval:\n",
    "        tags_eval[tag] = [[], [], [], []] # tp, tn, fp, fn\n",
    "\n",
    "    tn, fp = m[0]\n",
    "    fn, tp = m[1]\n",
    "\n",
    "    tags_eval[tag][0].append(tp)\n",
    "    tags_eval[tag][1].append(tn)\n",
    "    tags_eval[tag][2].append(fp)\n",
    "    tags_eval[tag][3].append(fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "NVxotdCp3VPg",
   "metadata": {
    "id": "NVxotdCp3VPg"
   },
   "source": [
    "Map fine-grained classes to actual classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "JGfL2S8HPyZ-",
   "metadata": {
    "id": "JGfL2S8HPyZ-"
   },
   "outputs": [],
   "source": [
    "classes = {'Person': 'PER', 'Judge': 'RR', 'Lawyer': 'AN',\n",
    "           'Country': 'LD', 'City': 'ST', 'Street': 'STR', 'Landscape': 'LDS',\n",
    "           'Organization': 'ORG', 'Company': 'UN', 'Institution': 'INN', 'Court': 'GRT', 'Brand': 'MRK',\n",
    "           'Law': 'GS', 'Ordinance': 'VO', 'European legal norm': 'EUN',\n",
    "           'Regulation': 'VS', 'Contract': 'VT',\n",
    "           'Court decision': 'RS',\n",
    "           'Legal literature': 'LIT'}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ArYsu9J3hfu",
   "metadata": {
    "id": "9ArYsu9J3hfu"
   },
   "source": [
    "Calculate Micro averaged performance metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "fz0lulc3RErG",
   "metadata": {
    "id": "fz0lulc3RErG"
   },
   "outputs": [],
   "source": [
    "for c in classes:\n",
    "    t = classes[c]\n",
    "    v = tags_eval[t]\n",
    "\n",
    "    precision = sum(v[0])/(sum(v[0]) + sum(v[2]))\n",
    "    recall = sum(v[0])/(sum(v[0]) + sum(v[3]))\n",
    "    f1 = 2 * ((precision * recall) / (precision + recall))\n",
    "\n",
    "    classes[c] = [round(precision*100, 2), round(recall*100, 2), round(f1*100, 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "OqWmU1tlRrOT",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OqWmU1tlRrOT",
    "outputId": "639c5da7-71bb-4a5f-d0ae-f924c8854812"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Brand': [83.13, 40.59, 54.55],\n",
       " 'City': [82.09, 90.91, 86.27],\n",
       " 'Company': [96.99, 93.88, 95.41],\n",
       " 'Contract': [95.36, 92.26, 93.78],\n",
       " 'Country': [91.67, 93.29, 92.47],\n",
       " 'Court': [99.11, 98.02, 98.56],\n",
       " 'Court decision': [99.11, 99.77, 99.44],\n",
       " 'European legal norm': [95.82, 97.54, 96.67],\n",
       " 'Institution': [92.86, 93.85, 93.35],\n",
       " 'Judge': [96.97, 98.6, 97.78],\n",
       " 'Landscape': [68.0, 45.95, 54.84],\n",
       " 'Law': [99.3, 99.51, 99.4],\n",
       " 'Lawyer': [86.96, 76.92, 81.63],\n",
       " 'Legal literature': [98.17, 96.03, 97.09],\n",
       " 'Ordinance': [98.86, 97.86, 98.36],\n",
       " 'Organization': [85.06, 91.39, 88.11],\n",
       " 'Person': [91.01, 88.36, 89.66],\n",
       " 'Regulation': [93.25, 94.7, 93.97],\n",
       " 'Street': [82.0, 75.93, 78.85]}"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "Fh7_VLa33ria",
   "metadata": {
    "id": "Fh7_VLa33ria"
   },
   "source": [
    "Finally, save our model for later use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "fzNxrAsHyVwI",
   "metadata": {
    "id": "fzNxrAsHyVwI"
   },
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), \"model.pt\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "german_bert_ner.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0079fc67b85841579d235bcc63039aba": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "032d826cd1ce4b82bc151ca9e957f6c5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "05597062f4a74a0c89ffe6371cbfc04b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_42d23d861ffb44baa2941e3640f01734",
      "placeholder": "​",
      "style": "IPY_MODEL_2c3bbdb526e148c2ac9f327aae1c353d",
      "value": " 29.0/29.0 [00:00&lt;00:00, 35.4B/s]"
     }
    },
    "0a0ac7e623834325830806e9a780c3b0": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "0ab85e4295174870a5541fe8fb0f6439": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0c37edd5fa00484485f0b891275cc0cd": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0da9cfa444d9431f9e4af04ecda9eaf7": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0ea4e6c53c504a95bde6cf2aff60ce24": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "17a4860b5c334a9a9816f16c9e540d47": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_7cd17bb918e94717a051ed6ab416cd51",
      "max": 29,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0ea4e6c53c504a95bde6cf2aff60ce24",
      "value": 29
     }
    },
    "1fc085f7317843eb871b60557b16f27a": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "240e9994af274a7c806360535c56f977": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0079fc67b85841579d235bcc63039aba",
      "max": 438869143,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_de3803d547a6450d8eeabf7bacab9e34",
      "value": 438869143
     }
    },
    "242defaf6f9c42bfa548ec67c27efa3c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "2c3bbdb526e148c2ac9f327aae1c353d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "372a3a41433b47a3bc271cf0f4e4ee5f": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3825314e00644a83af7564114c30d356": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ae5551d860794fb1b3fcf5325ec331ad",
      "max": 433,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_242defaf6f9c42bfa548ec67c27efa3c",
      "value": 433
     }
    },
    "3ca198d81a814391b45672b3fbf5cb70": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "40f855988bd045a18b54bd735288812d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_8134a304285b471cb8838397949d2fcb",
       "IPY_MODEL_d77a82b18daf4528a2a033cfb1dd2155"
      ],
      "layout": "IPY_MODEL_1fc085f7317843eb871b60557b16f27a"
     }
    },
    "42d23d861ffb44baa2941e3640f01734": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4929f2c059f948cdb526cce1acd74232": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d841d68e45764d85b3e7c29243421230",
      "placeholder": "​",
      "style": "IPY_MODEL_372a3a41433b47a3bc271cf0f4e4ee5f",
      "value": " 439M/439M [00:09&lt;00:00, 47.8MB/s]"
     }
    },
    "549ea47dba7343ceb80b52b64231906d": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "54b494a263f6486ab23bdf73cdfd107d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_032d826cd1ce4b82bc151ca9e957f6c5",
      "placeholder": "​",
      "style": "IPY_MODEL_a076cfbd889c483c9c73722ec21f8ba1",
      "value": " 433/433 [00:10&lt;00:00, 42.0B/s]"
     }
    },
    "7cd17bb918e94717a051ed6ab416cd51": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8134a304285b471cb8838397949d2fcb": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ca6504116ee849d098c7404eab27ba85",
      "max": 485115,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_0a0ac7e623834325830806e9a780c3b0",
      "value": 485115
     }
    },
    "869b0fb93fad468692c4edd48b5d8488": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_240e9994af274a7c806360535c56f977",
       "IPY_MODEL_4929f2c059f948cdb526cce1acd74232"
      ],
      "layout": "IPY_MODEL_549ea47dba7343ceb80b52b64231906d"
     }
    },
    "91f92d528872463faf922b67d4f04843": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a49f32a2a95041e8addc4cf61ee285d6",
      "placeholder": "​",
      "style": "IPY_MODEL_0da9cfa444d9431f9e4af04ecda9eaf7",
      "value": " 255k/255k [00:01&lt;00:00, 213kB/s]"
     }
    },
    "a076cfbd889c483c9c73722ec21f8ba1": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a49f32a2a95041e8addc4cf61ee285d6": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ae5551d860794fb1b3fcf5325ec331ad": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b808334ea7074334ac496201fe7983b8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "ca6504116ee849d098c7404eab27ba85": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cf23bbaa269c4140a7c4b2f03b6fe129": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_db7404baf0df442ebd3d552299fc2c40",
      "max": 254728,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b808334ea7074334ac496201fe7983b8",
      "value": 254728
     }
    },
    "d2c9b4da73e445c8a2a7cef1d1e0a6d6": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_17a4860b5c334a9a9816f16c9e540d47",
       "IPY_MODEL_05597062f4a74a0c89ffe6371cbfc04b"
      ],
      "layout": "IPY_MODEL_0c37edd5fa00484485f0b891275cc0cd"
     }
    },
    "d77a82b18daf4528a2a033cfb1dd2155": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0ab85e4295174870a5541fe8fb0f6439",
      "placeholder": "​",
      "style": "IPY_MODEL_e0ac7da8f2014ca79cb65e5bc4dd6436",
      "value": " 485k/485k [00:00&lt;00:00, 2.82MB/s]"
     }
    },
    "d8063e3b2ce74d88a6ee76e2eb374c08": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d841d68e45764d85b3e7c29243421230": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "db7404baf0df442ebd3d552299fc2c40": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "de3803d547a6450d8eeabf7bacab9e34": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "e0ac7da8f2014ca79cb65e5bc4dd6436": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ea67678edff04c4bae2d16685019e388": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cf23bbaa269c4140a7c4b2f03b6fe129",
       "IPY_MODEL_91f92d528872463faf922b67d4f04843"
      ],
      "layout": "IPY_MODEL_d8063e3b2ce74d88a6ee76e2eb374c08"
     }
    },
    "ead333525fcc4534ab7ce6bb8a828add": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3825314e00644a83af7564114c30d356",
       "IPY_MODEL_54b494a263f6486ab23bdf73cdfd107d"
      ],
      "layout": "IPY_MODEL_3ca198d81a814391b45672b3fbf5cb70"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}