examples/notebooks/02-active-learning-with-stopping-criteria.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "94536c6d",
   "metadata": {},
   "source": [
    "# Active Learning for Text Classification using Small-Text\n",
    "## 02 - Stopping Criteria\n",
    "\n",
    "This tutorial shows how to use stopping criteria when performing active learning for text classification using [small-text](https://github.com/webis-de/small-text). If you have read the previous notebook already, you can also directly jump to the [relevant parts for stopping criteria](#Stopping-Criterion).\n",
    "\n",
    "\n",
    "### Installation and Preparation\n",
    "\n",
    "Besides small-text, we also install [datasets](https://github.com/huggingface/datasets) to load an example dataset and [matptlotlib](https://matplotlib.org/) to plot the learning curves at the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cf5355d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: small-text[transformers]==1.0.0b2 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (1.0.0b2)\n",
      "Requirement already satisfied: scikit-learn>=0.24.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (1.0.1)\n",
      "Requirement already satisfied: dill in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (0.3.4)\n",
      "Requirement already satisfied: scipy in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (1.7.2)\n",
      "Requirement already satisfied: numpy>=1.20.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (1.21.4)\n",
      "Requirement already satisfied: tqdm in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (4.62.3)\n",
      "Requirement already satisfied: transformers>=4.0.0; extra == \"transformers\" in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (4.12.3)\n",
      "Requirement already satisfied: torch>=1.6.0; extra == \"transformers\" in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (1.10.0)\n",
      "Requirement already satisfied: torchtext>=0.7.0; extra == \"transformers\" in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from small-text[transformers]==1.0.0b2) (0.11.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from scikit-learn>=0.24.1->small-text[transformers]==1.0.0b2) (3.0.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from scikit-learn>=0.24.1->small-text[transformers]==1.0.0b2) (1.1.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (0.1.2)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (2021.11.10)\n",
      "Requirement already satisfied: filelock in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (3.3.2)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (0.10.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (21.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (6.0)\n",
      "Requirement already satisfied: sacremoses in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (0.0.46)\n",
      "Requirement already satisfied: requests in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (2.26.0)\n",
      "Requirement already satisfied: typing-extensions in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from torch>=1.6.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (3.10.0.2)\n",
      "Requirement already satisfied: pyparsing<3,>=2.0.2 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from packaging>=20.0->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (2.4.7)\n",
      "Requirement already satisfied: click in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from sacremoses->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (8.0.3)\n",
      "Requirement already satisfied: six in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from sacremoses->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (1.16.0)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0; python_version >= \"3\" in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (2.0.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (1.26.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5; python_version >= \"3\" in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests->transformers>=4.0.0; extra == \"transformers\"->small-text[transformers]==1.0.0b2) (3.3)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: datasets in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (1.15.1)\n",
      "Requirement already satisfied: matplotlib in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (3.4.3)\n",
      "Requirement already satisfied: seaborn in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (0.11.2)\n",
      "Requirement already satisfied: multiprocess in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (0.70.12.2)\n",
      "Requirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (6.0.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (4.62.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (1.21.4)\n",
      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (2021.11.0)\n",
      "Requirement already satisfied: aiohttp in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (3.8.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (2.26.0)\n",
      "Requirement already satisfied: dill in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (0.3.4)\n",
      "Requirement already satisfied: pandas in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (1.3.4)\n",
      "Requirement already satisfied: packaging in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (21.2)\n",
      "Requirement already satisfied: xxhash in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (2.0.2)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from datasets) (0.1.2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: cycler>=0.10 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from matplotlib) (0.11.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from matplotlib) (1.3.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from matplotlib) (8.4.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: scipy>=1.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from seaborn) (1.7.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (1.7.2)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (5.2.0)\n",
      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (2.0.7)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (21.2.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.1)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from aiohttp->datasets) (1.2.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (1.26.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
      "Requirement already satisfied: idna<4,>=2.5; python_version >= \"3\" in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from requests>=2.19.0->datasets) (3.3)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from pandas->datasets) (2021.3)\n",
      "Requirement already satisfied: pyyaml in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
      "Requirement already satisfied: filelock in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.3.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n",
      "Requirement already satisfied: six>=1.5 in /home/cschroeder/.local/share/virtualenvs/notebooks-R6UeU-dP/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install small-text[transformers]==1.1.0   # use \"small-text\" without \"[transformers]\" if you want to work on the CPU only\n",
    "\n",
    "# additional dependencies for this example\n",
    "%pip install datasets matplotlib seaborn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7813fe3",
   "metadata": {},
   "source": [
    "The code in the following cell sets the logging level and progress bars to make it suitable for notebooks. Moreover, we set a fixed random seed, so you get the same output every time you run this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cd157fbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets\n",
    "import torch\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import rcParams\n",
    "\n",
    "datasets.logging.set_verbosity_error()\n",
    "\n",
    "# disables the progress bar for notebooks: https://github.com/huggingface/datasets/issues/2651\n",
    "datasets.logging.get_verbosity = lambda: logging.NOTSET\n",
    "\n",
    "# set matplotlib params\n",
    "rcParams.update({'xtick.labelsize': 14, 'ytick.labelsize': 14, 'axes.labelsize': 16})\n",
    "\n",
    "# fix the random seed\n",
    "seed = 2022\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a52b467",
   "metadata": {},
   "source": [
    "### Loading and Preparing the Data\n",
    "\n",
    "Again, we load the rotten tomatoes dataset and build a TransformersDataset object which is then used for active learning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a894d142",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "from small_text import TransformersDataset\n",
    "\n",
    "\n",
    "\n",
    "raw_dataset = datasets.load_dataset('rotten_tomatoes')\n",
    "num_classes = np.unique(raw_dataset['train']['label']).shape[0]\n",
    "\n",
    "\n",
    "transformer_model_name = 'bert-base-uncased'\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    transformer_model_name\n",
    ")\n",
    "\n",
    "\n",
    "train = TransformersDataset.from_arrays(raw_dataset['train']['text'],\n",
    "                                        raw_dataset['train']['label'],\n",
    "                                        tokenizer,\n",
    "                                        max_length=60)\n",
    "test = TransformersDataset.from_arrays(raw_dataset['test']['text'], \n",
    "                                       raw_dataset['test']['label'],\n",
    "                                       tokenizer,\n",
    "                                       max_length=60)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e75ff407",
   "metadata": {},
   "source": [
    "### Active Learning\n",
    "\n",
    "Now we setup a tranformer-based active learning process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "92610a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "from small_text import (\n",
    "    PoolBasedActiveLearner,\n",
    "    PredictionEntropy,\n",
    "    TransformerBasedClassificationFactory,\n",
    "    TransformerModelArguments,\n",
    "    random_initialization_balanced\n",
    ")\n",
    "\n",
    "\n",
    "# simulates an initial labeling to warm-start the active learning process\n",
    "def initialize_active_learner(active_learner, y_train):\n",
    "    \n",
    "\n",
    "    indices_initial = random_initialization_balanced(y_train, n_samples=20)\n",
    "    active_learner.initialize_data(indices_initial, y_train[indices_initial])\n",
    "\n",
    "    return indices_initial\n",
    "\n",
    "\n",
    "\n",
    "transformer_model = TransformerModelArguments(transformer_model_name)\n",
    "clf_factory = TransformerBasedClassificationFactory(transformer_model, \n",
    "                                                    num_classes, \n",
    "                                                    kwargs=dict({'device': 'cuda', \n",
    "                                                                 'mini_batch_size': 32,\n",
    "                                                                 'class_weight': 'balanced'\n",
    "                                                                }))\n",
    "query_strategy = PredictionEntropy()\n",
    "\n",
    "active_learner = PoolBasedActiveLearner(clf_factory, query_strategy, train)\n",
    "indices_labeled = initialize_active_learner(active_learner, train.y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cdf91e0e",
   "metadata": {},
   "source": [
    "### Stopping Criterion\n",
    "\n",
    "Besides, we slightly change the evaluation: In order to make the setting more realistic, we do not evaluate on the test accuracy any longer, since you will not have a test set in real-world situations. \n",
    "\n",
    "Unlike before, we now also output the stopping response (True or False). The stopping criteria are not bound to the PoolBasedActiveLearner class because in real-world applications a user should have the final say whether the active learning process is stopped or not. Stopping criteria are intended to be an indicator, whose recommendation can be accepted or rejected."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8721e2ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train accuracy: 0.60\n",
      "Test accuracy: 0.49\n",
      "---------------\n",
      "Iteration #0 (40 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.57\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #1 (60 samples)\n",
      "Train accuracy: 0.83\n",
      "Test accuracy: 0.57\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #2 (80 samples)\n",
      "Train accuracy: 0.96\n",
      "Test accuracy: 0.59\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #3 (100 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.72\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #4 (120 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.67\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #5 (140 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.74\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #6 (160 samples)\n",
      "Train accuracy: 0.99\n",
      "Test accuracy: 0.76\n",
      "Stop: True\n",
      "---------------\n",
      "Iteration #7 (180 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.70\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #8 (200 samples)\n",
      "Train accuracy: 0.96\n",
      "Test accuracy: 0.76\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #9 (220 samples)\n",
      "Train accuracy: 0.98\n",
      "Test accuracy: 0.73\n",
      "Stop: False\n",
      "---------------\n",
      "Iteration #10 (240 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.79\n",
      "Stop: True\n",
      "---------------\n",
      "Iteration #11 (260 samples)\n",
      "Train accuracy: 0.93\n",
      "Test accuracy: 0.74\n",
      "Stop: True\n",
      "---------------\n",
      "Iteration #12 (280 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.74\n",
      "Stop: True\n",
      "---------------\n",
      "Iteration #13 (300 samples)\n",
      "Train accuracy: 0.98\n",
      "Test accuracy: 0.76\n",
      "Stop: True\n",
      "---------------\n",
      "Iteration #14 (320 samples)\n",
      "Train accuracy: 0.97\n",
      "Test accuracy: 0.80\n",
      "Stop: False\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from small_text import KappaAverage\n",
    "\n",
    "\n",
    "num_queries = 15\n",
    "stopping_criterion = KappaAverage(num_classes, kappa=0.9)\n",
    "\n",
    "\n",
    "def evaluate(active_learner, train, test):\n",
    "    y_pred = active_learner.classifier.predict(train)\n",
    "    y_pred_test = active_learner.classifier.predict(test)\n",
    "    \n",
    "    # Notice: We observe the train accuracy now.\n",
    "    train_acc = accuracy_score(y_pred, train.y)\n",
    "\n",
    "    print('Train accuracy: {:.2f}'.format(train_acc))\n",
    "    print('Test accuracy: {:.2f}'.format(accuracy_score(y_pred_test, test.y)))\n",
    "    \n",
    "    return train_acc\n",
    "\n",
    "\n",
    "results = []\n",
    "stopping_history = []\n",
    "\n",
    "results.append(evaluate(active_learner, train[indices_labeled], test))\n",
    "stopping_history.append(stopping_criterion.stop(predictions=active_learner.classifier.predict(train)))\n",
    "\n",
    "\n",
    "for i in range(num_queries):\n",
    "    # ...where each iteration consists of labelling 20 samples\n",
    "    indices_queried = active_learner.query(num_samples=20)\n",
    "\n",
    "    # Simulate user interaction here. Replace this for real-world usage.\n",
    "    y = train.y[indices_queried]\n",
    "\n",
    "    # Return the labels for the current query to the active learner.\n",
    "    active_learner.update(y)\n",
    "\n",
    "    indices_labeled = np.concatenate([indices_queried, indices_labeled])\n",
    "    \n",
    "    print('---------------')\n",
    "    print(f'Iteration #{i} ({len(indices_labeled)} samples)')\n",
    "    results.append(evaluate(active_learner, train[indices_labeled], test))\n",
    "    \n",
    "    stopping_criterion_response = stopping_criterion.stop(predictions=active_learner.classifier.predict(train))\n",
    "    print(f'Stop: {stopping_criterion_response}')\n",
    "    stopping_history.append(stopping_criterion_response)\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cbadf91",
   "metadata": {},
   "source": [
    "### Visualization\n",
    "\n",
    "You can use the learning curve plot, which now depicts the learning curve of the training accuracy, to visualize the earliest iteration at which the stopping criterion returned `True`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7801fcc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvoAAAH0CAYAAABM/Yk4AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABpFElEQVR4nO3deXxcdb3/8dcne9NmabN2BQpNFyoUKBQKtCCtIlcURNnR6lUU3Ld71Z/3uuv1el24XpFFvRUsQkXRC8pSUFqgLbRAQUg32tIFOtMkbWeSJpP1+/vjTGoIWSbJTM4s7+fjkUc6c86c+cw0bd7nO5/z/ZpzDhERERERSS9ZfhcgIiIiIiLxp6AvIiIiIpKGFPRFRERERNKQgr6IiIiISBpS0BcRERERSUMK+iIiIiIiaSjH7wLS0YUXXugeeughv8sQEUmYP1z7BwDe85v3+FyJiEjGs/42KOgnQH19vd8liIgkVHhf2O8SRERkEGrdERERERFJQ2kR9M1skZn9n5m9ZmbOzJbF8Ji3mNlqM2uJPu7fzcx67XOZmdWaWWv0+6UJexEiIiIiInGUFkEfGAe8BHwaaBlsZzMrBlYBQeD06OO+CHyuxz5nAfcAK4B50e+/M7MFca5dRERERCTu0qJH3zn3F+AvAGa2PIaHXAMUAh9wzrUAL5nZLOBzZvYj55wDPgP8zTn3nehjvmNm50fvvyq+r0BEJLVMOWuK3yWIiMgg0iLoD8NZwBPRkN/tYeBbwLHArug+P+31uIeBT4xGgSIiyWzJ95b4XYKIiAwiXVp3hqoar22np2CPbQPtU00fzOx6M9toZhvr6uriVqiIiIiIyHBkatCPO+fcbc65+c65+RUVFX6XIyKSUCsvW8nKy1b6XYaIiAwgU1t3AkBVr/uqemwbaJ8AIiIZrrmh2e8SRERkEJk6or8OONfMCnrctxR4HXi1xz5Lez1uKbA24dWJiIiIiIxQWgR9MxtnZvPMbB7ea5oWvT0tuv17ZvZYj4fcBTQDy81srpm9B/gS0D3jDsBNwFvN7EtmNsvMvgycD/xklF6WiIiIiMiwpUXQB+YDz0e/xgDfiP75m9HtE4Hju3d2zoXwRucnARuBnwE/BH7UY5+1wJXAMuBF4P3AFc65pxP7UkRERERERi4tevSdc48DNsD2ZX3c93dg0SDHvRe4d4TliYikneMuOM7vEkREZBBpEfRFRGR0Lf63xX6XICIig0iX1h0REREREelBQV9ERIZsxTtWsOIdK/wuQ0REBqDWHRERGbL2lna/SxARkUFoRF9EREREJA1pRF9ERGJW+3qYu57ZzaFXD5KbncXP/vYKl506heqSgsEfLCIio0pBX0REBtTS1skDL77OXc/s4fk9h8nPyeJ9hXm0dXbxg4e38sNHtrK4poLL50/lgtlV5OXow2IRkWSgoC8iIn3aHmxkxdN7+MNz+whHOji+Yiz/9s45XHbqZGpLSwD4+AdP5t5n93Hvs/u4YcVzTBibx6WnTOby+VOZWV3k8ysQEcls5pzzu4a0M3/+fLdx40a/yxARGbLWjk4eeinAivV7eObVg+RlZ3Hh3GquWTCNM46bgFnfaxN2djme2F7H7zbu45HaAO2djpOnlHD56VO5+ORJFBfkjvIrERHJGP0uGqugnwAK+iKSanbWNfHbZ/Zw77P7ONTczjFlhVx9xjTee9oUysblD+lYB4+08cfnX2Plxr1sCTRSkJvFO+ZO5PL5U1lw3ASysvr9nSQiIkOnoD+aFPRFJBW0dXSxqjbIiqd3s3ZHAzlZxttOrOLqM45h4fFlAwby5ectB2DZ48v63cc5x99fC7Fy417+tOl1GiMdTJtQyPtOm8J7509hYsmYOL8iEZGM1O9/1urRFxHJMHsPNvPbZ/awcuNe6pvamFw6hi++fSbvmz+FyqL4zZ5jZpw0pZSTppTy1X+aw0MvBVi5cS8/XLWNHz+6jXNneBfwLplTSX5OdtyeV0REPAr6IiIZoKOzi8e2HGDF03t4YnsdBrx1VhXXnDmNRTMqyE5wO01BbjaXnDKZS06ZzN6Dzfxu417ufXYfH7/rOcYX5nJJ9ALe2ROLE1qHiEgmUdAXEUljrx9u4e4Ne7lnwx6C4Vaqiwv41FtncOUZU31rnZk6oZDPvW0mn15Sw1Ov1HPPxr2sWL+H/33qVU6aUsL75k/lXSdPomSMLuAVERkJBX0RkTTT2eVYve0Adz29h79uOYADFtdU8K13T+OtsyrJyU6Oee6zs4xFNRUsqqng0JE2/rTpNe7ZuI9/++NLfPuBWt4xt5rL50/lzOkDXy8gIiJ908W4CaCLcUXED8FwhJUb9nL3hr28driF8nH5XHH6FK48fRpTJxTG9bk23LwBgNNvPD2uxwV4KXoB7x+ff41wpIOpE8bwvtOmctlpU5hcqgt4R0NTawert9axdkc9WWYUFeQwriCHooJcigtyGJfv/bko+ufiglzG5mcnzUmkSIbRrDujSUFfREZLV5fjyVfquevpPazaHKSzy3HOCeVcvWAaS+dUkZvCwSvS3snDLwf43cZ9PPlKPWZwzgnlXHH6VJbOqdIFvHEWDEdYVRtkVW2QdTsaaOvsoig/h+xsozHSQWfX4HmhMC/7aPjvPhEoKsihKD/3DScLRfnR+wtyo/f9Y7+C3Kx+12uQzBBqaWfT3sM8t/sQz+89zKY9hygqyGVRTTmLZlSw8IRytfa9kYL+aFLQ71tbRxc765uYVa2L7URGqr6pld9t3Mdvn9nDnoPNTBibx/tOm8KVZ0zjuPKxCX/+9uZ2AHILR+eX7d6DzUdX4H3tcAulhblcMs+7gHfOJP2fMhzOObYFm1hVG2BVbZAX9oUAOLaskKVzqlg6p5rTjhlPdpbhnCPS3kVjpJ3G1g4aIx00Rtppinh/DkfaaYre3xTpoLG1PbpPdL/otua2zkHrysnq8QlCvnciUNx9UtDrBKG4IIfZE4upqdIqzKmqq8vxSl0Tz+0+xHN7DvH8nsNsP9AEQJZBTVURp0wrpaGpjbU7Gmhq7SA7y5g3tZRFMypYVFPOSVNKEz6hQJJT0B9NCvp9u2fDHr78h7/zty+cxzFliQ8iIunGOce6nQ3c9fQeHn7ZW332jOMmcM2CaVw4t3pUR7hjmUc/Ebq6HGt3NHDPxr08/HKAto4u5k4u5vL5U3n3yZMpGaUTj1TV0dnFhlcPsao2yKObg+w52AzAvKmlLJ1TxdvmVHFC5biEjah3dHZxpLWTcMQ7EfBOALwTgXCvk4fubeE+Th56frqQm23ccu1pXDC7KiE1S3yFmtt5bq8X6J/fc4hNew7T2NoBQGlhLqdOG88pU0s59ZjxnDy1lHH5/7ictL2zi+f3HGbNtjrWbK/j76+FcM573DknlHvX/MyooLokftMEpwgF/dGkoN+37z+0hZ8/voNvvvtE3n/WsX6XI5IyDh1p4/fP7eOuZ/aws+4IxQU5XHbaFK5ZMI0TKv0ZyfQr6Pd0uLmN/3vhdVZu3MtLr4XJy8niwhO9C3gHW/Arkxxp7WDNtjpW1Qb569YDHG5uJy8ni7OPL2PpnGqWzK6ksjh1gtHRTxda2znc3M4XfvcCW/Y3cuv7T+P8mZV+lyc9dHY5th9o5Lndh6Oj9YfYUXcE8EbrZ1YXc+q0Ui/cTyvluPKxQzrJPHikjSe217FmWz1rttdR19gKwMyqIq/Np6aC04+dQEFu2rf5KeiPJgX9vn3unk384fnXuGBWJb9cFv8L+ETSiXOOjbsPcdfTe/jz3/fT1tHFqdNKuWbBMfzTSRN9/8WVDEG/p5dfD/G7jfu47/nXCLW0M7l0DO85dTKnTCtl9sRiqosLMqrv+0BjhEdrD7CqNsBTOxpo6+iiZEwuF8yqZOmcKhbVVDA2Pz0m3gs1t3P1L9az/UATt79/PotrKvwuKWMdOtLm9dbv8dpwXtgboik6Wj9hbN7RkfpTppVy8pTSuP4MOufYEmg8Otq/Ydch2jq7KMjNYsFxZSyqqWBxTTnHVyTuEysfKeiPJgX9vl19+3rW7mhgTG42m762VBfSieC1ohxobOW1w828djjCa4daeO1wM8/sOsi2YBPj8nO49JTJXL1gWlItJpVsQb9bpL2TVbVBVm7cy5Ov1NP9K660MJdZ1UXMqi5mzsRiZk0soqaqyPcTpnhxzvHKgSYeiV5Mu2nvYQCmThjD0tnVLJ1TxenHjk/bWXEOHWnj6l88zc66Jn75gdM5Z0a53yWlvc4ux9ZAI8/vPcRzu702nJ313mh9dpYxq7ro6Ej9qdPGc0xZ4agG7Oa2DtbvbPBG+7fVHa1tcumYdLyoV0F/NCno9+2tP3yc+sZWwpEOVnx4AWefoP+I041zjv2hCKGWdsrG5jF+bF5Kz/oSD5H2Tl4/3MJrh1u874davEB/uJnXDrcQCEVo73zj/8MlY3KZUTmO9542hYtPnpSUI6/JGvR7Ckfa2RZoZPP+MJuj37cGGo9eEJplcFz5WGZNjIb/6iJmTyxmYklqjP53djme3X3o6MW0rzZ4/fYnTSlh6ewqlp5YxcyqopR4LfFw8EgbV9++nlcbjvCrZaez8Hj9jomng0faeH7PPy6YfWHvYY5E/y2Vjc3jlGnjOfWYUk6ZOp6Tp5ZQmJdc/2/tPdjMmu11rNlWx9pXGmhMr4t6FfRHk4L+mznnOPFrD/Oukyfx++f28cGzj+MrF832uywZgY7OLnbUHaF2f4ja18O8/HqY2v1hDkdnY+lWMiaXsnF5lI3NY8LYPMrG5VM2Nnp7XD7lY/OYMC6PsrH5jC/MTakRR+ccoZZ29h1qORrmXzvUwuuh7kDfQn1T2xsek2VQVVzA5NIxTB4/hkmlY47+eXKpd3tcEgb73jYt3wTAvGXzfK1jqLq6HHsONrMlEKZ2fyNb9ofZHAiz92DL0X1KxuQeDf2zJ3qfAtRUFTEmz//R/5a2TtZsj/bbbznAwSNt5GYbZx1f7s2UM7sqEy9EPKqhqZWrbl/P3oMtLP/g6SyYXuZ3SSmpo7OLrcFGnttzmOejs+F0n0hmZxlzJhYfHak/ddp4pk4Yk1InlO2dXWzaG72od1sdL/a4qPfsE8pZPMNbyC+F/i0p6I8mBf03C0faOenrj/CVi2bx+NY6GpraePizi/wuS2J0pLWDLYFomI8G+i2BRto6ugDIz8liVnURcyZ5I6MTxuZzsLmNhqZWDh5po6GpjYYjrTQ0tXHwSBsHm9vo678eMygdk+udEIzNp2xcr5OD6O3ycflMGJvH+MK8hI6+dHY5guHI0dH4fYfeODL/+uGWoyNa3Qpys/4R3nuE+O77qksKMv5TjmTUGGlna6Dx6Mj/lujPeM/R/2PLx3rhP9oCNHtSMZNGYfS/vqmVxzZ7LTlPbK+ntaOLooIc3hrtt19cU0FRQVq0H8RFXWMrV962jv2hCHd86AzmHzvB75KSXkNTqxfqoyP2L+4LHf3ZLx+Xz6nTSr0R+2mlnDSlNClOeuOp50W9T2yv40DqXdSroD+aFPTfbHuwkaU/XsNNV84jEIrwvQe3sP7LF6TS2XLGOBCO8PL+aKCPhvpXG468odf5xGigP3FSCXMmFTO9fOyQRuI7uxyHm73QXx8N/90nAg1HWt94f1Mrh1va+z0xGF8YPRmIngiUjc2PngzkMSF6slAWPVkoHZP7hplYWto6vVH4N7TV/GNkPhCOvGmRoPGFuV5wL/nHKHzPEfkJY/NSamRruJrrvdG9wvL4rribTLq6HHsPNXutP/sb2RLwvndPSQlQXJDDrGj4nz2xmFkTi5kZh9H/HXVNRxevem7PIZzzeou9+e2rOOO4CTphHMCBcIQrb1tPMBzhjn9ewGnHjPe7pKTU0dnFd/+yhV89tQvw1jCYM6n4Db31U8an1mj9SKXoRb0K+qNJQf/Nnthex3W/fIZ7rj+TksJcLvzJE/znZSdx+elT/S4tY3V2OV5tOPKGUfra18PUN7Ue3WfqhDGcOLHk6Ej9iZP9mb2ko7OLQ83tR4N/w5G2N/y5+5OC+uhJQu/2oW5Z0ROD0sLco8frKTvLqI621UwqLYiG90ImlRYwJToqn2x9p35JhR79RGlq7WBrNPRvjo78b9kfPvrpjhkcV+aN/s86egJQxOTS/gNTZ5dj095DRy+m3RmdgvDEScVHw/2cicXJFCySXiAU4crb1tHQ1MadH17AvKmlfpeUVMKRdj551/Os3lbH1Qumcekpk5k7qSTtRutHqrmtg6d3HmR1NPh3/9ucVFLgzdtfU8HZ/l/U2+9/DPqNJaMiEIoAUF1SwLQJhVQV57N6W52C/ihpaetka7AxGuhDvPx6mC37G2lp94JJbrYxo7KI82ZWHB2tnzWx2O//uI7Kyc6ioiifiqJ8YPB549s7uzjU3OMEINpC1P1JweHmNkoL86LhvYDJpYVMHj+GqqL8lLpGQPwxLj+H046ZwGnH/KMlpKvLse9QS7StLczm/WFeej3En/++/+g+RQU5zK72Qn/3SUBDUxuraoM8tiVIfVMbOVnGmdPL+MBZx7JkThWTS8f48RLTQnVJAb+9/kyuuHU91/3yaVZ8eAEnTSn1u6yksLvhCP/86428Wn+E773nLVx1xjS/S0pahXk5nD+rkvNneWs07D3YzBPbvZl8/vzifu7esJcs8xad+8a75vKWKSU+V/xGCvoyKoJhL+hXRUeDF9dU8NBLATo6uxSs4uzgkTZefj30hlH6HXVNdHegFBXkMGdiMVeeMZU5E4uZM6mYGZVF5OWkz99DbnYWlUUFVBapNUxGR1aWMa2skGllhVw4t/ro/d7of/fIv/cpwO+f3feGazuK8nNYPLOCpXOqOG9mZdKcYKeDiSVjomF/Hdf+4mnu+siZzJ2cXEFstK3f2cDHfvMsAHf+8wLOOl4XLA/F1AmFXL1gGlcvmPami3pLk3BlbgV9GRWBcITxhblHL2RZXFPJyo37eGHf4TeMiknsuvuHuwN9dwtOIHpSBd5Hi3MmFfOOt0yM9tQXZ1y/pYifvNH/8W/oEe/qcrx22Bv9L8zLZsFxZWl1op1sJpeO4bcfOZMrb1vPtb98mrs+fCZzJiXPmhSj6Z4Ne/jqH19i2oRCfvmB0zm2fKzfJaW03OwsTj92AqcfO4HPv22m3+X0SUFfRkUgFKGqxxLr55xQTpbB6q11CvpD0NzWwY8e2caL+0LU7g8fXXEwO8s4oWIcZx1fdnSUfs7EYsaPzfO5YhHpLSvLmDqhkKkT0vdC5mQzdUIhv/3ImVxx2zqu+cV6fnv9mcyqzpyw39nl+N5fNvOLJ3dx7oxy/ufqU/XJUYZQ0JdREQhH3jDDTklhLvOmlrJ6Wx2fS9Kz4GT05xf384sndzFvaimXnjLZ66efVJxWK3xKaph/w3y/SxAZkmllPcL+7U9z9/VnMqNq8Gt+Ul1jpJ1P/fZ5/ra1jmULj+Wr/zRbLbMZREFfRkUg1MrcSW/si1xcU8lPHtvGwSNtTNDIc0xW1QaZWFLAfTcuVPuN+GruFXP9LkFkyI4tHxsN++u5Khr2T6gc53dZCbP3YDP//OsN7Kg7wrcvmcu1Zx7jd0kyynRKJwnX3tlFw5HWN7TuACyeWYFz3tSbMrhIeydPbK9nyewqhXzxXWhviNDekN9liAzZ9Ipx/PYjZwJw9e3r2VnX5HNFibHh1YO8+2dPEYguHKaQn5kU9CXhDjS24hxM7LU41lsmlzC+MJfV2xT0Y/Hk9npa2jtZOqfK71JEuO+6+7jvuvv8LkNkWE6oHMddH1lAZ5fjqtvX82r9Eb9LiqvfbdzL1bevp2RMLn/8+NmcfUK53yWJTxT0JeG659Cv6hX0s7OMc2dUsGZbPV1dWrhtMKtqgxTl53DmdE2FJiIyUjVVRaz4yALaO72wv6ehefAHJbnOLsf3HtzMF+99kTOOm8Afbzyb6RXp25okg1PQl4Q7ulhW8ZvnNF9cU0F9Uyu1+8OjXVZK6exyPLYlyOKZFZqGT0QkTmZVF/Obf15AS3snV92+nr0HUzfsN7V28NE7n+XW1Tu59sxpLP/gGZQk4bzuMrqUGCThuud17yvon1vjfZyo9p2Bbdp7mPqmNrXtiIjE2ZxJXthvjLRz1e3ree1wi98lDdm+Q8289+dr+euWIN9414l8+5K3kKuZdQQFfRkFwXCEvJysPleMqywq4MRJxQr6g1hVGyQnyzhvZqXfpYiIpJ25k0v4zYcXEGpp56rb1rM/lDph/9ndh7jkZ0/x2uEWln/wDD6w8Fi/S5IkoqAvCRcIRaguLuh3pphFNRU8t/sQ4Uj7KFeWOlbVBlgwfYIWOJGkcdbnz+Ksz5/ldxkicXPSlFLu/OcFHDrSxlW3rSfYY5XxZHXf8/u46rb1jM3P4b4bz2ZRTYXfJUmSSZugb2Y3mtkuM4uY2bNmdu4g+3/czDabWYuZbTWz9/favszMXB9fb+4/kQEFwpE+23a6La6poKPLsfaVhlGsKnXsrGtiR90Rls5W244kj5kXz2TmxVrsTtLLvKmlLP/QGdQ1tnLVbes5kKRhv6vL8YOHt/DZe17g1GNK+eONZ6f1egAyfGkR9M3sCuAm4LvAKcBa4EEzm9bP/jcA3we+CZwIfA34mZld3GvXZmBizy/nXHL+q09iwV6r4vZ26rTxjMvPYY3m0+/TqtogAEvUny9JpH5rPfVb6/0uQyTuTjtmPMs/dAaBcISrbl9PXWOr3yW9QXNbBzeseJaf/W0HV50xlTs+tIDxWnRS+pEWQR/4HLDcOXe7c26zc+6TwH7ghn72vw643Tn3W+fcTufc3cBtwL/22s855wI9vxL3EtKTc85r3Rkg6OflZLHw+DJWb63DOU2z2duq2iBzJhYzZXyh36WIHPXARx/ggY8+4HcZIglx+rET+N9lp/P64QhX376e+qbkCPuvH27hvT9fx6raIP/+zjl899K3aCY2GVDK/3SYWR5wGvBIr02PAAv7eVg+0HtkvgU4w8x6NkGPMbPdZrbPzB4ws1PiUnQGOdzcTmtH15tWxe1t8cwKXjvcwo669Fq0ZKQamlp5ds8hzbYjIjLKFkwv41fLTmfvoWau/cXTHDzS5ms9z+85xLt/9hR7Djbzy2Wn86FzjtMq6TKolA/6QDmQDQR73R8Eqvt5zMPAh8zsdPPMBz4M5EaPB7AV+BDwbuAqvBODp8xsRl8HNLPrzWyjmW2sq1MLSreBptbsadEM7wIizb7zRo9tOYBzKOiLiPjgrOPL+OUHTmdX/RGu+cXTHG72J+z/adNrXHHbegpys/jDjQs5XzOwSYzSIegPx7eAP+P18rcDfwJ+Hd3WBeCcW+ec+7VzbpNz7gngCmAH8Mm+Duicu805N985N7+iQle9dzsa9EvyB9xv6oRCjq8Yq6Dfy6raIJNKvClIRURk9J19Qjm3v38+O+qauPaXTxNqHr0Z4rq6HD9atY1P372JeVNK+dPHz6GmqmjUnl9SXzoE/XqgE+g95FkF9NlT75xrcc59CCgEjgWmAa8CjUCfSdM51wlsBPoc0Ze+BaOr4g7WugOwuKaSp3c2EGnvTHRZKaGlrZMnttexZE6VPp4VEfHRopoKbr32NLYFmrjuV08Takl82G9p6+QTv32O/35sO+87bQq/+fACJuiiWxmilA/6zrk24Flgaa9NS/FG7Ad6bLtzbl80xF8JPOCc6+prX/OS1kl4F/lKjALhCGbewliDWVRTTmtHF+t3appNgCdfqSfS3qW2HUlKi766iEVfXeR3GSKj5vxZldx8zals3h/mA796hsYErv0SCEW4/NZ1PPhSgP930Wz+870n6aJbGZZ0+an5EbDMzD5sZrPN7CZgEnALgJndYWZ3dO9sZjVmdp2ZzTCzM8zsbmAu8JUe+3zNzN5uZtPNbB7wS7ygf8sovq6UFwxHKBubH9N/UGdOLyM/J0vtO1GragMU5eew4Lgyv0sReZPpS6Yzfcl0v8sQGVVL5lTxs6tP5aXXQiz73w00tXbE/Tle3HeYd/3Pk+ysa+IX75/PRxZN16e6MmxpEfSdc/cAnwG+CmwCzgEucs7tju4yLfrVLRtvSs4XgFVAAbDQOfdqj31K8abc3Iw3g89kYJFz7pkEvYy05E2tOXB/freC3GwWTC9jjYI+nV2OxzYf4LxZlRrFkaQU2BQgsEkzDkvmeduJ1fzP1aewae9hPvi/z3AkjmH/gRdf5/Jb15GbncXvb1zIBVooUUYobRKEc+5m59yxzrl859xpzrk1Pbad55w7r8ftzc65U5xzhc65EufcJc65rb2O91nn3DHR41U6597unFs3ii8pLewPDbwqbm+LayrYUXeEvQebE1hV8tu09xANR9rUtiNJ66HPPMRDn3nI7zJEfHHh3In895Wn8Nyew3xw+Qaa20YW9p1z3PTodj5x1/PMnVTCnz5xNrOqNQmDjFzaBH1JTsFwJKYLcbstrvFmLMr0VXIfqQ2Sk2VH3w8REUku/3TSRH50+clsfPUg/7x8Iy1tw5tIItLeyafu3sSPH93Ge06dzIqPLKB8XGyfhIsMRkFfEibS3smh5vYhjegfXzGWyaVjWL01s4P+qtogZ04vo2RM7uA7i4iIL949bzI/vPxk1u9q4CN3bBzyrHEHwhGuuHUdD7z4Ol96xyx++L6Tyc/JTlC1kokU9CVhDoS9JcOrSmIP+mbG4pkVrN3RQFtHnxMgpb0ddU3srDuith0RkRRw6SlT+MF7T+apHfV89M5nYw77L70W4l3/8xTbDzRx67Wn8bHFx+uiW4k7BX1JmO7FsiYOIeiD177T1NrBc3sOJaKspLeq1lvkeYmCvohISnjvaVP4j/e8hdXb6rhxxXO0dgwc9h96aT/vu2UdWQb3fmwhbzuxepQqlUyT43cBkr6Oroo7hNYdgIXHl5GTZazeVseZ0zNvaslVtUFOnFTM5NIxfpci0q8LvnuB3yWIJJUrTp9GZxd85b6/8/EVz3HzNae9adY05xw3P76DHzy8lVOmlXLrdafFtM6MyHBpRF8SJhBqAYbWugNQVJDLqceMz8g+/fqmVp7bc0htO5L0pi6cytSFU/0uQySpXL1gGt+6ZC6Pbj7AJ+56jvbOf7SgRto7+ew9m/jBw1u5ZN4kfvuRMxXyJeEU9CVhAqFWCvOyKcof+gdHi2sqqN0f5kBjJAGVJa+/bj6Ac7BEcydLktu7di971+71uwyRpHPdmcfw9Yvn8EhtkE/99nnaO7uoa2zlqtvX88dNr/PFt8/kx1fMoyBXF91K4ql1RxImGPbm0B/OxUWLayr4wcNbeWJbPZedNiUB1SWnR2qDTC4dw4mTNH+yJLfHvvIYAMseX+ZvISJJaNnZx9HR5fj2nzfzsTufZfP+MIea27nl2lO5cO5Ev8uTDKIRfUmYwBDn0O9pzsRiysflszqDVsltaevkyVfqWDK7UjMviIikuA+fO52vXDSLx7YcoMvB7z52lkK+jDqN6EvCBEIRFhw3YViPzcoyFtWU87ctB+jscmRnpX/wfWJ7HZH2LpbO0ewLIiLp4PpFxzN3cgk1VUVaBEt8oRF9SYiuLseBxsiQL8TtaXFNBYea2/n7a6E4Vpa8VtUGKSrIYcH04Z0ciYhI8ll4fLlCvvhGQV8S4mBzG+2dbshTa/Z07owKzMiI2Xc6uxx/3XKA82dWkputf5YiIiIycmrdkYQIhLzZcobbow8wYWweJ00pZfW2A3x6yYx4lZaUnt9ziIYjbVokS1LGhT+50O8SRERkEBo6lIToDvrVI2jdAVg8o5xNew8Tam6PR1lJa1VtkNxs47yZFX6XIhKT6nnVVM/T9SQiIslMQV8SYrir4va2eGYFXQ6efKU+HmUlrVW1Qc6cXkZxQa7fpYjEZOejO9n56E6/yxARkQEo6EtCBMMRsgzKx+WN6DgnTymluCCH1dsOxKmy5PPKgSZ21h/RariSUtZ8ew1rvr3G7zJERGQACvqSEIFQhMqiAnJGeGFpTnYW586oYPW2OpxzcaouuayqDQJaDVdERETiS0FfEiIQHtnUmj0trqkgGG5la7AxLsdLNo9uDjJ3cjGTSsf4XYqIiIikEQV9SYhAKEJ1cXzmDV5U412gmo7TbNY1tvLcnkMazRcREZG4U9CXhAiEIyO+ELdbdUkBs6qLWL0t/YL+X7cEcQ7154uIiEjcaR59ibvmtg4aIx1xa90Br33nV0/t4khrB2Pz0+fHdlVtkMmlY5gzsdjvUkSG5J23vtPvEkREZBAa0Ze4OzqHfpxG9MFr32nvdKzb0RC3Y/qtua2DJ7bXs3ROFWbmdzkiQ1I+s5zymeV+lyEiIgNQ0Je4i9cc+j3NP3Y8Y3KzWbM9fdp3ntheT2tHl9p2JCVtvX8rW+/f6ncZIiIygPTpgZCkEQzHZ1XcnvJzsll4fFla9ek/WhukqCCHM46b4HcpIkO27ofrAJh58UyfKxERkf5oRF/iLhBqBeIb9MFbJXd3QzOv1h+J63H90Nnl+OuWA5w/s5LcEa41ICIiItIXJQyJu0CohaKCHArz4vuB0eLuaTbTYFT/uT2HaDjSprYdERERSRgFfYm7eE6t2dMxZWM5tqwwLYL+qtogudnGeTMr/C5FRERE0pSCvsRdINwa97adbotrKli3o4FIe2dCjj8anHOsqg1y5vQyigpy/S5HRERE0pSCvsRdMBShKgEj+uD16be0d7Lx1UMJOf5o2FHXxK76I7xNbTuSwi6981IuvfNSv8sQEZEBKOhLXHV2OeqaWpmYoBH9M6eXkZedxeptBxJy/NGwqtarfYmCvqSwkqkllEwt8bsMEREZgIK+xFV9UyudXS5hI/qFeTmcftx41myrT8jxR8Oq2gBvmVzCxJIxfpciMmwv3fMSL93zkt9liIjIABT0Ja72J2BV3N4W11SwNdjI/lBLwp4jUeoaW3l+72GWzNZovqS2jT/fyMafb/S7DBERGYCCvsRVIBT/xbJ6W1xTCcCaFJx957HNQZxD02qKiIhIwinoS1x1r4qbqNYdgJqqcVQXF6TkNJuraoNMLh3D7IlFfpciIiIiaU5BX+IqEI6Qm22Ujc1L2HOYGYtrKnhiez0dnV0Je554a27r4MlX6lk6pwoz87scERERSXMK+hJXwVCEyqICsrISG2QXz6ygMdLBpr2HE/o88fTE9npaO7o0raaIiIiMihy/C5D0EghHEtqf3+3sE8rJzjJWb6tj/rETEv588bCqNkhxQQ6nH5ca9YoM5PJ7L/e7BBERGYRG9CWuAuFIQmfc6VYyJpdTppamTJ9+Z5fjr1sOcP6sSnKz9c9OUl9heSGF5YV+lyEiIgNQ4pC4cc4RSOCquL0tqqng76+FaGhqHZXnG4lndx/i4JE2zbYjaWPT8k1sWr7J7zJERGQACvoSN42tHTS3dVJdkj8qz7e4pgLn4MlXkn/xrFW1AXKzvYuIRdKBgr6ISPJLm6BvZjea2S4zi5jZs2Z27iD7f9zMNptZi5ltNbP397HPZWZWa2at0e+XJu4VpL5gKPFTa/b0lsklTBibx+qtyd2+45xjVW2Qs44vp6gg1+9yREREJEOkRdA3syuAm4DvAqcAa4EHzWxaP/vfAHwf+CZwIvA14GdmdnGPfc4C7gFWAPOi339nZgsS90pSWyCc+FVxe8rKMs6dUc6a7XV0dblRec7h2FHXxKsNzWrbERERkVGVFkEf+Byw3Dl3u3Nus3Puk8B+4IZ+9r8OuN0591vn3E7n3N3AbcC/9tjnM8DfnHPfiR7zO8Dj0fulD92r4k4sGTNqz7m4poL6pjZq94dH7TmH6pHaIABLZlf6XImIiIhkkpQP+maWB5wGPNJr0yPAwn4elg9Eet3XApxhZt29FWf1ccyHBzhmxuteFbeyeHR69AHOneH1vCfz7DuraoO8ZXLJqJ4AiYiIiKR80AfKgWwg2Ov+IFDdz2MeBj5kZqebZz7wYSA3ejyij435mGZ2vZltNLONdXXJGzoTaX8owvjCXApys0ftOSuK8pk7uThp+/QPNEbYtPew2nYk7Vzzl2u45i/X+F2GiIgMIB2C/nB8C/gzXi9/O/An4NfRbV3DOaBz7jbn3Hzn3PyKisycWSUYHr2pNXtaXFPBs3sOEY60j/pzD+axzQdwDgV9STu5hbnkFurichGRZJYOQb8e6AR6J6kqINDXA5xzLc65DwGFwLHANOBVoBHoHhoODOWYMnqr4va2aEYFnV2Ota80jPpzD2ZVbZAp48cwq7rI71JE4mrDzRvYcPMGv8sQEZEBpHzQd861Ac8CS3ttWoo3Yj/QY9udc/ucc53AlcADzrnuEf11wzlmJguEWkdtxp2eTj1mPOPyc5KuT7+5rYMnX6ln6ZwqzMzvckTi6uWVL/Pyypf9LkNERAaQ43cBcfIj4E4zewZ4CvgYMAm4BcDM7gBwzr0/ersGWACsB8bjzdozF/hAj2PeBKwxsy8BfwQuBc4Hzkn8y0k97Z1dNBxp9aV1Jzc7i7NPKGPNtjqcc0kTqtdsq6eto0ttOyIiIuKLlB/RB3DO3YM37eVXgU14Yfwi59zu6C7Tol/dsvHC/QvAKqAAWOice7XHMdfijfIvA14E3g9c4Zx7OnGvJHUdaGzFOZjoQ+sOwOKaSl473MKOuiZfnr8vq2qDFBfkcPqxE/wuRURERDJQuozo45y7Gbi5n23n9bq9GW9hrcGOeS9wbzzqS3fdc+hX+RT0F9V4kyU9vrWOEyr974fv6Ozir1uCvHVWJbnZaXE+LSIiIilGCUTiojvo+9GjDzBlfCEnVI5Lmj79Z3cf4lBzO0vn9DfDq4iIiEhipc2IvvgrEPY36IM3zead63fT0tbJmLzRm8u/L6tqg+RlZ7F4ZmZOtSrpb9njy/wuQUREBqERfYmLYDhCXk4WpT7Oq724poK2ji7W7/J3mk3nHKs2Bznr+DLG5etcWkRERPyhoC9xEQhFqC4u8HXGmzOOm0B+ThZrfG7feeVAE7sbmjXbjqS1tf+1lrX/pdmGRUSSmYK+xEUgHPG1bQegIDebM6eX+d6n/0htEIAlsxX0JX1te2Ab2x7Y5ncZIiIyAAV9iYugT6vi9ra4poKddUfYe7DZtxpW1QY5aUpJUrwfIiIikrkU9GXEnHPsDyVJ0I9e/OrXqP6BcIRNew+zVKP5IiIi4jMFfRmxw83ttHV0+bIqbm/Ty8cyZfwY34L+o5sPALD0RAV9ERER8ZeCvoxYMkyt2c3MWFxTwdpX6mnr6Br15390c5CpE8Yws8r/RbtEEil3TC65Y/ybZUtERAanoC8jdjTol+T7XIlncU0FR9o6eXb3oVF93iOtHTz5Sj1LZ1f7OvuQyGi45sFruObBa/wuQ0REBqCgLyMWjK6KmwytOwALTygnJ8tGvX3nie11tHV0sWRO5ag+r4iIiEhfFPRlxALhCGZQWZQcQX9cfg6nHTN+1OfTf6Q2SMmYXM44dsKoPq+IH1Z/azWrv7Xa7zJERGQACvoyYsFwhLKx+eTlJM+P0+KZFdTuD3Mg2laUaB2dXfx1ywHeOquSnOzkeR9EEmXXY7vY9dguv8sQEZEBKJHIiHlTayZHf363xTXeNJtrttePyvNt3H2Iw83tWg1XREREkoaCvoxYIOT/qri9zZlYTEVR/qj16T9aGyQvO4tF0RMMEREREb8p6MuIBcORpLkQt5uZsWhGBU9sr6OzyyX0uZxzrNocZOEJZYzLz0noc4mIiIjESkFfRiTS3smh5vakG9EHr0//cHM7L+47nNDn2X6gid0NzSzRariSQQrLCiksK/S7DBERGYCGH2VEDoRbAagqSb6gf+4J5ZjB6m11nDJtfMKeZ1VtEED9+ZJRLv/95X6XICIig9CIvoxI92JZE5Mw6I8fm8fJU0oT3qf/SG2Qk6eUJF37koiIiGQ2BX0Zkf2hFoCkbN0Bb/adF/Ye5nBzW0KOHwxHeGHvYY3mS8Z59MuP8uiXH/W7DBERGYCCvoxIMDqin4ytOwCLairocvDkK4mZZvOxzQcAWDqnOiHHF0lW+9btY9+6fX6XISIiA1DQlxEJhFopzMumKElnmzl5SgklY3JZvTUx7TuragNMm1BITdW4hBxfREREZLgU9GVEgmFvDn0z87uUPuVkZ3HOjHJWb6vDufhOs3mktYOndjSwZHZV0r5+ERERyVwK+jIigSScQ7+3xTUVHGhsZUugMa7HXbOtjraOLvXni4iISFJS0JcRCYQiSTnjTk+Lo6vVxnv2nVW1QUoLczn92MRN3SmSrIqnFFM8pdjvMkREZADJ2VgtKaGry3GgMZK0F+J2qyouYFZ1Eau31vGxxcfH5ZgdnV38desB3jqzkpxsnS9L5nnPb97jdwkiIjIIJRQZtoYjbbR3uqSdWrOnxTMr2Lj7IEdaO+JyvI27D3G4uV1tOyIiIpK0FPRl2I5OrZkKQb+mgvZOx7odDXE53qraIHnZWSyKtgWJZJqHPvMQD33mIb/LEBGRASjoy7AFQl7Qr07y1h2A+cdMoDAvOy59+s45VtUGWXhCGWOTdFpRkUQLbAoQ2BTwuwwRERmAgr4MWyA6op8KrTt5OVksPL6Mx7cdGPE0m9uCTew52Ky2HREREUlqCvoybMFwhCyD8nF5fpcSk8U1Few92MKrDc0jOs6qWm8Uc8lsBX0RERFJXgr6MmyBUITKooKUmXVmcU0lAKu3HhjRcVbVBjl5amlKXJsgIiIimSs1EpokpUA4+afW7GlaWSHHlY8dUZ9+MBzhhX0h3qa2HclwZTVllNWU+V2GiIgMQFcSyrAFQhGmV4z1u4whWVxTwd0b9hBp76QgN3vIj390cxBA/fmS8S6+7WK/SxARkUFoRF+GLRCOpMSFuD0trqkg0t7FhlcPDuvxq2qDHFNWyIzKcXGuTERERCS+FPRlWJrbOmiMdKRU6w7AgukTyMvJYs0w2neaWjtY+0oDS2ZXYWYJqE4kddx//f3cf/39fpchIiIDUNCXYTk6h36KjegX5uVwxrEThtWnv2ZbHW2dXWrbEQEatjXQsC0+C9CJiEhiKOjLsBydQz/FRvTBa9/ZFmzi9cMtQ3rcqtogpYW5zD9mfIIqExEREYkfBX0ZlmAKLZbV2+KZFQBDat/p6Ozir1sO8NZZlSkznaiIiIhkNiUWGZb9odQd0Z9ROY6JJQVDat/Z8OohQi3tmlZTREREUkbaBH0zu9HMdplZxMyeNbNzB9n/ajPbZGbNZhYws9+YWXWP7cvMzPXxlXrJNgGCoQhFBTkU5qXeDK1mxuKaCp7cXk97Z1dMj1lVGyQvJ4tzZ1QkuDqR1FA9r5rqedWD7ygiIr5Ji6BvZlcANwHfBU4B1gIPmtm0fvY/G7gT+DVwInAJMAdY0WvXZmBizy/nXCQBLyHlpOLUmj0trqmgsbWDTXsPD7qvc45VmwOcfXwZY/NT78RGJBEu/MmFXPiTC/0uQ0REBpAWQR/4HLDcOXe7c26zc+6TwH7ghn72PwvY55z7sXNul3NuPfBTYEGv/ZxzLtDzK3EvIbUEwq0p2bbTbeEJ5WRnWUx9+luDjew92MLSORq9FBERkdSR8kHfzPKA04BHem16BFjYz8OeAiaa2cXmKQeuBP7Sa78xZrbbzPaZ2QNmdkpci09hwVCEqhQe0S8Zk8up00pj6tNf9bK3Gu6S2ZWJLkskZfzh2j/wh2v/4HcZIiIygJQP+kA5kA0Ee90fBPocgnXOrcML9iuANqAOMOADPXbbCnwIeDdwFRABnjKzGX0d08yuN7ONZraxrm7oc7Snko7OLuqaWpmYwiP64LXvvLgvRH1T64D7Pbo5yLyppVSm8ImNSLyF94UJ7wv7XYaIiAwgpqBvZt/tr989FZnZHLxWnW/hfRpwId5Jwa3d+zjn1jnnfu2c2+ScewK4AtgBfLKvYzrnbnPOzXfOza+oSO8LNuub2ujscik9og+wqMb7e3pye32/+wTDEV7YF9IiWSIiIpJyYh3R/ySw08z+YmbvMrNk+iSgHugEeiexKqC/nvovA884537gnHvROfcwcCNwnZlN6esBzrlOYCPQ54h+Jgmk8Bz6Pc2dVMKEsXkDtu+sqvU+KNK0miIiIpJqYg3sE4GP44XnPwK7zexrZjY5UYXFyjnXBjwLLO21aSne7Dt9KcQ7Oeip+3af74mZGXAS3kW+GS2QwnPo95SVZSyaUc6abXV0dbk+91lVG+SYskJOqBw3ytWJiIiIjExMQd851+Scu9U5dxrezDSPAF8EdpnZfWbm9xxrPwKWmdmHzWy2md0ETAJuATCzO8zsjh773w+828xuMLPp0ek2/xt4zjm3J/qYr5nZ26Pb5wG/xAv6t4zi60pK3avipnrrDnir5DYcaePl19/ca9zU2sG6HQ0snV2Fd54nIt2mnDWFKWf1+QGoiIgkiSFPCu6c2wBsMLMvAb/Du1j1XWa2G/gh8HPnXGyrEMWJc+4eMysDvor36cNLwEXOud3RXab12n+5mRUBn4jWHAL+Cvxrj91KgdvwevdDwPPAIufcMwl8KSkhEI6Qm22Ujc3zu5QR614Aa/W2A7xlSskbtq3eWkdbZ5f680X6sOR7S/wuQUREBjHkoG9mxwMfBZYB44H78AL/xcBPgJOB6+NWYYycczcDN/ez7bw+7vsp3gW5/R3vs8Bn41VfOgmGIlQWFZCVlfqj3OXj8nnL5BLWbKvnE2994+UXj24OMr4wl9OOGe9TdSIiIiLDF+usO9lm9l4zW4U37eQ1wM+BY5xzlznn7nbOXYN30e4ViStXkkEgHEn5/vyeFtdU8OyeQ4Qj7Ufva+/s4q9bDvDWWVXkZCfTteciyWHlZStZedlKv8sQEZEBxJpgXgPuwfsE4Cq8gP8159zrvfZ7HiiKY32ShAKhSMrPuNPT4pkVdHY51r7yj2k2N7x6kFBLu9p2RPrR3NBMc0Oz32WIiMgAYg36K4G5zrnznXO/c8519LWTc+5p55yGP9OYc45AOLVXxe1t3tRSivJz3jDN5qraIHk5WZw7o9zHykRERESGL6YefefcpxJdiKSGxtYOmts6qS7J97uUuMnNzuLsE8pZvbUO57xpNlfVBjnnhHLG5g/5MhYRERGRpBBrj/6/mlmfF66a2X+b2RfjW5Ykq2AofabW7GnxzApeD0V45UATWwKN7DvUorYdERERSWmxDld+EG8ayr5sAr4A/CAeBUlyS5dVcXtbVNM9zWYdLW2dmMEFsyt9rkokeR13wXF+lyAiIoOINehPA7b3s20ncEx8ypFk170q7sSSMT5XEl+TS8cwo3Icq7fVEWppZ97UUiqL0utkRiSeFv/bYr9LEBGRQcR64WwzMLmfbVOA1viUI8muO+hXFqdPj363xTUVrN/ZwIv7QmrbERERkZQXa9B/Aviimb0h3UVvfz66XTJAIBxhfGEuBbnZfpcSd4tnVtDe6V2Mu3S2gr7IQFa8YwUr3rHC7zJERGQAsbbufB1YC2wzs9/gzas/GbgWKMNbJVcyQDDNptbs6fRjJ1CQm0V1cQEnVI7zuxyRpNbe0j74TiIi4qtYp9d8wczOB/4L+Fe8TwK6gCeBy5xzLySuREkm6bYqbk8Fudn8y9tnUVVcgJn5XY6IiIjIiMQ8Sbhz7hlgkZmNAcYDh5xzLQmrTJJSINTK3EklfpeRMB86RzOJiIiISHoY8mpA0XCvgJ+B2ju7aDjSmratOyIiIiLpJOagb2Z5wDuAmUDvpOecc9+KZ2GSfA40tuIcTEzT1h0RiV3NO2v8LkFERAYRU9A3s0l4/fjHAg7obmB2PXZT0E9zgZD3QU6Vgr5Ixlv4hYV+lyAiIoOIdXrNHwB1eAtnGbAAmA58B3gl+mdJc4GQt1xCuq2KKyIiIpKOYm3dORf4AvB69HaXc+5V4N/NLBv4b+Dd8S9Pkkkg7C2WpaAvIsvPWw7AsseX+VqHiIj0L9YR/TLgdedcF3AEb9adbn8FzotzXZKEguEIeTlZlBbm+l2KiIiIiAwi1qC/DyiP/nkH8LYe284AIvEsSpJTIBShWnPMi4iIiKSEWFt3/gYsBv4I3Ar8zMzmAe3A26P3SZpL58WyRERERNJNrEH/q8AEAOfcz80sB7gCKAT+E/hmYsqTZBIIRZg3tdTvMkREREQkBrEG/XZgd/cN59xPgZ8mpCJJSs45jeiLyFEnXn6i3yWIiMggBg360dH7BuBS4P6EVyRJ6XBzO20dXVoVV0QAOP3G0/0uQUREBjHoxbjOuQ4gCHQmvhxJVppaU0R6am9up7253e8yRERkALHOuvMb4MOJLESS29GgX5LvcyUikgxWXLSCFRet8LsMEREZQKw9+q8CV5vZBuBPwH7A9dzBOfer+JYmySQY8oK+WndEREREUkOsQf9n0e+TgdP62O4ABf00FghHMIPKIgV9ERERkVQQa9A/LqFVSNILhCKUjc0nLyfWbi8RERER8VNMQd85t3vwvSSdeVNrqj9fREREJFXEOqIvGS4QijBl/Bi/yxCRJDFv2Ty/SxARkUHEFPTNbBe9Lr7tzTk3PS4VSVIKhiOcdsx4v8sQkSShoC8ikvxiHdFfzZuDfhmwEGgC/hrPoiS5RNo7OdTcrjn0ReSo5vpmAArLC32uRERE+hNrj/6yvu43s1LgIeDR+JUkyeZAuBWAqhIFfRHxrHzvSgCWPb7M30JERKRfI5pCxTl3GPgB8O9xqUaS0v5QCwATFfRFREREUkY85kqMAFPicBxJUkdXxVXrjoiIiEjKGPasO2aWA8wFvg68HK+CJPkEo0FfrTsiIiIiqSPWWXe66H/WnTDwT3GrSJJOINRKYV42RfmajVVEREQkVcSa3L7Jm4N+BNgNPOicC8W1KkkqwXCE6uICzMzvUkQkScy/Yb7fJYiIyCBinXXn6wmuQ5JYIByhSv35ItLD3Cvm+l2CiIgMIqaLcc2swsxq+tlWY2bl8S1LkkkgFNGMOyLyBqG9IUJ79WGuiEgyi3XWnZuBz/ez7bPR7b4ysxvNbJeZRczsWTM7d5D9rzazTWbWbGYBM/uNmVX32ucyM6s1s9bo90sT+yqST1eXIxiO6EJcEXmD+667j/uuu8/vMkREZACxBv1zgIf72fYIcHZ8yhkeM7sCuAn4LnAKsBZ40Mym9bP/2cCdwK+BE4FLgDnAih77nAXcE71vXvT778xsQaJeRzJqONJGR5fT1JoiIiIiKSbWoD8e6O8z2jBQFp9yhu1zwHLn3O3Ouc3OuU8C+4Eb+tn/LGCfc+7Hzrldzrn1wE+BniH+M8DfnHPfiR7zO8Dj0fszxtGpNRX0RURERFJKrEF/H28MwT0twAvVvjCzPOA0vE8WenoEWNjPw54CJprZxeYpB64E/tJjn7P6OObDAxwzLQVC0cWy1LojIiIiklJiDfr3Al82szfMlx+9/SVgZbwLG4JyIBsI9ro/CFS/eXdwzq3DC/YrgDagDjDgAz12qx7KMc3sejPbaGYb6+rqhvoakpZWxRURERFJTUOZR38R8H9mFgBeAybjhd71wDcSU15imNkcvFadb+GN0k8EfgDcCrx/OMd0zt0G3AYwf/78/hYXSznBcIQsg/JxeX6XIiJJ5KzPn+V3CSIiMohY59FvNrPFwHXAUrye/FfwWlt+45zrSFyJg6oHOoGqXvdXAYF+HvNl4Bnn3A+it180syPAE2b2Fefcvuhjh3LMtLQ/FKGyqICc7Fg//BGRTDDz4pl+lyAiIoOIdUQf51w78KvoV9JwzrWZ2bN4JyC/67FpKfD7fh5WiHdy0FP37e5Euy56jB/02Gcp3ow+GUNTa4pIX+q31gNQPlPLqIiIJKuYgr6ZnQlMc869qRffzN4H7HHOPR3v4obgR8CdZvYM3oW2HwMmAbcAmNkdAM657rac+4HbzewG/tG68xPgOefcnug+NwFrzOxLwB+BS4Hz8aYazRiBUITpFWP9LkNEkswDH30AgGWPL/O3EBER6Ves/Rjfw5tvvi+zo9t945y7B2/ay68Cm/DC+EXOud3RXaZFv7r3X443JecngJfwLjbeBry7xz5r8S7YXQa8iNe7f4XPJzSjLhCO6EJcERERkRQUa+vOycB/9rPtGeBT8Sln+JxzN9PPCr3OufP6uO+neBfkDnTMe/FOAjJSc1sHjZEOte6IiIiIpKBYR/QLBtg3G1BvRxo6Ooe+RvRFREREUk6sQX8z8K5+tr0L2BqfciSZHJ1DXyP6IiIiIikn1tadW4BbzSwM3I63Uu5k4Hrgn4EbE1Oe+Ekj+iLSn0VfXeR3CSIiMohY59G/3cxmAp/Fu4j16Cbgx9HFoiTNaERfRPozfcl0v0sQEZFBDGUe/S+Y2c+BJXgLZtUDjzrndiaqOPFXMBShqCCHwryYf0xEJEMENnlrB1bPq/a5EhER6c+QEpxzbgewI0G1SJLR1Joi0p+HPvMQoHn0RUSS2ZCHas2sEm8WnjfosdCUpIlAuFVtOyIiIiIpKtaVcbOAbwMfBUr72S07TjVJkgiGIsyo1PL2IiIiIqko1uk1PwN8HPghYMB38YL/LrxWno8kojjxT0dnFwcaI0zUiL6IiIhISoo16H8Q+Cbw/ejt+5xzXwNmA68B0xJQm/iovqmNLgdV6tEXERERSUmx9uhPBzY65zrNrAMYA+CcazeznwA/Bb6ekArFF0en1lTQF5E+XPDdC/wuQUREBhFr0A/xjwtwXwdmAk/1OMaEONclPju6WJZad0SkD1MXTvW7BBERGUSsQf95YA7wcPTrG2bWAnQA3wGeS0x54pdgdERfrTsi0pe9a/cCCvwiIsks1qD/E7z2HYCvAacCK6K3dwOfiG9Z4rdAOEJutlE2Ns/vUkQkCT32lccAzaMvIpLMYgr6zrlVPf4cMLMzgOOBQmCzc649QfWJT4KhCJVFBWRlmd+liIiIiMgwDHnBLADnnANeiXMtkkT2hyLqzxcRERFJYbFOrykZJhiOaMYdERERkRSmoC9v4pwjEI7oQlwRERGRFDas1h1Jb42tHTS3dVJdku93KSKSpC78yYV+lyAiIoNQ0Jc3CYY0taaIDKx6XrXfJYiIyCDUuiNvolVxRWQwOx/dyc5Hd/pdhoiIDGBII/pmVg1M4x+r5B7lnFsTr6LEX/ujI/oTS8b4XImIJKs13/b+y5++ZPoge4qIiF9iCvpmNhm4E1jc12bAAdlxrEt81N26U1msHn0RERGRVBXriP7PgbcA/wL8HWhNWEXiu0A4wvjCXApyde4mIiIikqpiDfrnAp9yzt2ZyGIkOQQ1taaIiIhIyov1YtwW4EAiC5HkEQhrVVwRERGRVBfriP7twHXAwwmsRZJEINTK3EklfpchIknsnbe+0+8SRERkELEG/deA68zsMeBB4GDvHZxzv4pnYeKPto4uGo60qnVHRAZUPrPc7xJERGQQsQb9W6LfjwXO72O7AxT008CBxgjOwUS17ojIALbevxWAmRfP9LkSERHpT6xB/7iEViFJIxhdLKtKQV9EBrDuh+sABX0RkWQWU9B3zu1OdCGSHAIhb+ZUrYorIiIiktpinXVHMkQgOqKvoC8iIiKS2vod0TezncClzrkXzGwXXh9+f5xz7vi4VyejLhiOkJeTRWlhrt+liIiIiMgIDNS6sxoI9/jzQEFf0kQgFKG6uAAz87sUERERERmBfoO+c+6DPf68bFSqEd8FQlosS0QGd+mdl/pdgoiIDCLWWXckQwTCEeZNLfW7DBFJciVTtaieiEiyG1LQN7OTgZnAm4Z8nXN3xKso8YdzjkBYI/oiMriX7nkJgLlXzPW5EhER6U9MQd/MSoE/A2d23xX93rNvX0E/xR1ubqeto0ur4orIoDb+fCOgoC8iksxinV7zu0AZsAgv5F8KvBVYAewEzkhIdTKqNLWmiIiISPqINei/HS/sr4/e3uece9w5937gUeDTiShORtfRoF+S73MlIiIiIjJSsQb9icBO51wnEAGKemz7A/BP8S5sqMzsRjPbZWYRM3vWzM4dYN/lZub6+DrSY5/z+tln1ui8otEXDHlBX607IiIiIqkv1qAfAEqjf94NnNVj2wnxLGg4zOwK4Ca8Tx1OAdYCD5rZtH4e8mm8k5eeXzuBlX3se2Kv/bbHtfgksj8UwQwqixT0RURERFJdrLPuPIl3Ie4DwJ3A18zsWKAD+ADwfwmpLnafA5Y7526P3v6kmV0I3AB8uffOzrkQEOq+bWZnA9OB6/o49gHnXH38S04+wXCEsrH55OXEev4nIpnq8nsv97sEEREZRKxB/xvApOiff4B3Ye4VQCFeyP9k/EuLjZnlAacB/9Vr0yPAwhgP8xHgZefc2j62bTSzfKAW+LZz7m/DLjbJeVNrqj9fRAZXWF7odwkiIjKImIZunXM7nHNPRP/c7pz7vHNuinNugnPuaudcQ2LLHFA5kA0Ee90fBKoHe7CZlQCXA7f32rQf7xOBy4D3AFuBx/rr/Tez681so5ltrKurG9orSBKBUEQz7ohITDYt38Sm5Zv8LkNERAYwaNA3szwzO2hm7xqNgnxwLd77cGfPO51zW51ztzjnnnXOrXPO3Qg8BHyxr4M4525zzs13zs2vqKhIfNUJEAxHdCGuiMREQV9EJPkNGvSdc214vfiRxJczLPVAJ1DV6/4qvIuIB/MR4PfOuYMx7Ps0MGNo5aWGSHsnh5rbNaIvIiIikiZiveryj8B7E1jHsEVPRJ4FlvbatBRv9p1+mdkZwMm8uW2nP/PwWnrSzoFwKwDVJQr6IiIiIukg1otxHwT+28zuxQv9+wHXcwfn3F/jW9qQ/Ai408yeAZ4CPoZ38fAtAGZ2B0B0ga+erge2O+ce731AM/sM8CrwMpCH1+JzCV7PftrZH2oBFPRFRERE0kWsQf/30e/viX51c4BFv2fHsa4hcc7dY2ZlwFfx5rp/CbjIObc7usub5tM3syLgSuCb/Rw2D2+GoSlAC17g/yfn3F/iXH5SOLoqrlp3RERERNJCrEH/rfQawU82zrmbgZv72XZeH/c1AuMGON5/Av8Zr/qSXTAa9Ks0oi8iMbjmL9f4XYKIiAwipqDfV2uLpJdAqJXCvGyK8mM99xORTJZbmOt3CSIiMoiYLsY1s51mdnI/2+aa2c74liWjLRj25tA3M79LEZEUsOHmDWy4eYPfZYiIyABinXXnWKC/JVMLgGPiUo34JqA59EVkCF5e+TIvr3zZ7zJERGQAsQZ96L9Hfz5weOSliJ8CoQgT1Z8vIiIikjb6bcg2s88Cn43edMD9ZtbWa7cxwATg7sSUJ6Ohq8t5q+Iq6IuIiIikjYGuvNwJPBb98weAjUBdr31agVrgF/EvTUZLw5E2OrqcptYUERERSSP9Bn3n3J+APwHdF2h+0zm3a5TqklF0dGpNBX0RERGRtBHr9JofTHQh4p9AKLpYllp3RCRGyx5f5ncJIiIyiKFcjCtpSqviioiIiKQfBX0hGI6QnWVUFPU3g6qIyBut/a+1rP2vtX6XISIiA1DQF/aHIlSMyyc7S4tliUhstj2wjW0PbPO7DBERGYCCvmhqTREREZE0pKAvBEIRqovVtiMiIiKSThT0hUA4ogtxRURERNJMTNNrSvpqbuugMdKh1h0RGZLcMbl+lyAiIoNQ0M9wR+fQ14i+iAzBNQ9e43cJIiIyCLXuZDgtliUiIiKSnhT0M5wWyxKR4Vj9rdWs/tZqv8sQEZEBKOhnuKNBXyP6IjIEux7bxa7HdvldhoiIDEBBP8MFQxGKCnIozNPlGiIiIiLpREE/w2lqTREREZH0pKCf4QLhVrXtiIiIiKQh9WtkuGAowozKcr/LEJEUU1hW6HcJIiIyCAX9DNbR2cWBxggTNaIvIkN0+e8v97sEEREZhFp3Mlh9UxtdDqrUoy8iIiKSdhT0M5jm0BeR4Xr0y4/y6Jcf9bsMEREZgFp3MphWxRWR4dq3bp/fJYiIyCA0op/BgtERfbXuiIiIiKQfBf0MFghHyM02ysbm+V2KiIiIiMSZgn4GC4QiVBYVkJVlfpciIiIiInGmHv0MFghF1J8vIsNSPKXY7xJERGQQCvoZLBiOMHuiflmLyNC95zfv8bsEEREZhFp3MpRzjkA4ogtxRURERNKUgn6GamztoLmtk+qSfL9LEZEU9NBnHuKhzzzkdxkiIjIAte5kqGBIU2uKyPAFNgX8LkFERAahEf0MpVVxRURERNKbgn6G2h8d0Z9YMsbnSkREREQkERT0M1R3605lsXr0RURERNKRevQzVCAcYXxhLgW52X6XIiIpqKymzO8SRERkEAr6GSqoqTVFZAQuvu1iv0sQEZFBpE3rjpndaGa7zCxiZs+a2bkD7LvczFwfX0d67bc4eqyIme00s48l/pWMjkBYq+KKiIiIpLO0CPpmdgVwE/Bd4BRgLfCgmU3r5yGfBib2+toJrOxxzOOAv0SPdQrwPeCnZnZZgl7GqAqEWjXjjogM2/3X38/919/vdxkiIjKAdGnd+Ryw3Dl3e/T2J83sQuAG4Mu9d3bOhYBQ920zOxuYDlzXY7ePAa875z4Zvb3ZzBYAXwB+H/+XMHraOrqob2rViL6IDFvDtga/SxARkUGk/Ii+meUBpwGP9Nr0CLAwxsN8BHjZObe2x31n9XHMh4H5ZpY7nFqTxYFGzaEvIiIiku5SPugD5UA2EOx1fxCoHuzBZlYCXA7c3mtTdT/HzIk+Z+/jXG9mG81sY11dXYyl+yMYXSyrSiP6IiIiImkrHYL+SF2L9z7cOZKDOOduc87Nd87Nr6ioiE9lCRIItQIa0RcRERFJZ+nQo18PdAJVve6vAgIxPP4jwO+dcwd73R/o55gd0edMWYGwWndEZGSq5w36gamIiPgs5YO+c67NzJ4FlgK/67FpKYNcNGtmZwAnA5/pY/M64NJe9y0FNjrn2oddcBIIhiPk5WRRWpjSlxqIiI8u/MmFfpcgIiKDSJfWnR8By8zsw2Y228xuAiYBtwCY2R1mdkcfj7se2O6ce7yPbbcAk83sJ9FjfhhYBvxXQl7BKAqEIlQXF2BmfpciIiIiIgmS8iP6AM65e8ysDPgq3pz4LwEXOed2R3d503z6ZlYEXAl8s59j7jKzi4Af403T+TrwKedcSk+tCdGgrwtxRWQE/nDtHwB4z2/e43MlIiLSn7QI+gDOuZuBm/vZdl4f9zUC4wY55mrg1HjUl0wC4Qjzppb6XYaIpLDwvrDfJYiIyCDSpXVHYuScIxDWiL6IiIhIulPQzzCHm9tp6+iiSjPuiIiIiKQ1Bf0Mo6k1RURERDJD2vToS2yOBv2SfJ8rEZFUNuWsKX6XICIig1DQzzCBkBf01bojIiOx5HtL/C5BREQGodadDBMIRTCDyiIFfREREZF0pqCfYYLhCGVj88nL0V+9iAzfystWsvKylX6XISIiA1DrTobxptZUf76IjExzQ7PfJYiIyCA0rJthAqGIZtwRERERyQAK+hkmGI7oQlwRERGRDKCgn0Ei7Z0cam7XiL6IiIhIBlCPfgYJHp1DX0FfREbmuAuO87sEEREZhIJ+BumeQ19BX0RGavG/Lfa7BBERGYRadzLI0VVx1bojIiIikvYU9DNId+tOlUb0RWSEVrxjBSvescLvMkREZABq3ckggVArhXnZFOXrr11ERqa9pd3vEkREZBAa0c8gwbA3h76Z+V2KiIiIiCSYgn4G2R9q0Rz6IiIiIhlCQT+DBMOtTFR/voiIiEhGULN2hujqct6quAr6IhIHNe+s8bsEEREZhIJ+hmg40kZHl9PUmiISFwu/sNDvEkREZBBq3ckQR6fWVNAXERERyQgK+hlCq+KKSDwtP285y89b7ncZIiIyAAX9DKFVcUVEREQyi4J+hgiEImRnGRVF+X6XIiIiIiKjQEE/QwTCESrG5ZOdpcWyRERERDKBgn6G0NSaIiIiIplF02tmiEAowvSKsX6XISJp4sTLT/S7BBERGYSCfoYIhCMsPL7M7zJEJE2cfuPpfpcgIiKDUOtOBjjS2kFjpEOtOyISN+3N7bQ3t/tdhoiIDEBBPwNoak0RibcVF61gxUUr/C5DREQGoKCfAYJaLEtEREQk4yjoZwCN6IuIiIhkHgX9DHA06GtEX0RERCRjKOhngGAoQlFBDoV5mmRJREREJFMo+WWAQDiith0Riat5y+b5XYKIiAxCQT8DBMKtatsRkbhS0BcRSX5q3ckAgVCLRvRFJK6a65tprm/2uwwRERmAgn6a6+jsoq5RI/oiEl8r37uSle9d6XcZIiIyAAX9NFff1EaXgyqN6IuIiIhklLQJ+mZ2o5ntMrOImT1rZucOsn+emX0z+phWM9tjZp/qsX2Zmbk+vlIqMWsOfREREZHMlBYX45rZFcBNwI3Ak9HvD5rZHOfcnn4edjcwBbge2A5UAWN67dMMHN/zDudcJI6lJ1xAq+KKiIiIZKS0CPrA54Dlzrnbo7c/aWYXAjcAX+69s5m9DbgAON45Vx+9+9U+juucc4EE1DtqgtERfbXuiIiIiGSWlG/dMbM84DTgkV6bHgEW9vOwS4ANwOfMbJ+ZbTez/zazcb32G2Nmu6P7PGBmp8S1+FGwPxQhN9soG5vndykikkbm3zCf+TfM97sMEREZQDqM6JcD2UCw1/1BYEk/j5kOnAO0ApcBpcBPgUnAe6P7bAU+BLwAFAGfBp4ys5Odc9t7H9DMrsdrA2LatGnDfzVxFgxHqCwqICvL/C5FRNLI3Cvm+l2CiIgMIh2C/nBkAQ642jkXAjCzTwAPm1mVcy7onFsHrOt+gJmtBTYBnwQ+1fuAzrnbgNsA5s+f7xL+CmIUCEXUny8icRfaGwKgZGqJz5WIiEh/Ur51B6gHOvEupu2pCuivv34/8Fp3yI/aHP3e53C8c64T2AjMGH6poy8YjmjGHRGJu/uuu4/7rrvP7zJERGQAKR/0nXNtwLPA0l6blgJr+3nYU8CkXj35NdHvu/t6gJkZcBLeSUJKcM4RCEd0Ia6IiIhIBkr5oB/1I2CZmX3YzGab2U14/fa3AJjZHWZ2R4/97wIagP81sxPN7Gy86Tnvdc4diD7ma2b2djObbmbzgF/iBf1bRu9ljUxjawfNbZ1Ul+T7XYqIiIiIjLK06NF3zt1jZmXAV4GJwEvARc657tH5ab32bzKzJXgX4G4ADgF/BL7UY7dSvJ77aiAEPA8scs49k7hXEl/BkKbWFBEREclUaRH0AZxzNwM397PtvD7u2wq8bYDjfRb4bLzq88P+kFbFFREREclUaRP05c0C0cWyJpb0XvBXRGRkzvr8WX6XICIig1DQT2PdrTuVxerRF5H4mnnxTL9LEBGRQaTLxbjSh0A4wvjCXApys/0uRUTSTP3Weuq31vtdhoiIDEBBP40FNbWmiCTIAx99gAc++oDfZYiIyAAU9NNYIKxVcUVEREQylYJ+GguEtCquiIiISKZS0E9TbR1d1De1aURfREREJEMp6KepA42aQ19EREQkk2l6zTQVjM6hX6URfRFJgEVfXeR3CSIiMggF/TQVCLUCGtEXkcSYvmS63yWIiMgg1LqTprpXxVXQF5FECGwKENgU8LsMEREZgIJ+mgqGI+TlZFFamOt3KSKShh76zEM89JmH/C5DREQGoKCfpvZHp9Y0M79LEREREREfKOinqWBIi2WJiIiIZDIF/TQVCGuxLBEREZFMpqCfhpxzXtDXiL6IiIhIxtL0mmnocHM7bR1dVGlEX0QS5ILvXuB3CSIiMggF/TSkqTVFJNGmLpzqdwkiIjIIte6koUAoGvRL8n2uRETS1d61e9m7dq/fZYiIyAAU9NPQ0RH9kjE+VyIi6eqxrzzGY195zO8yRERkAAr6aSgQimAGlUUa0RcRERHJVAr6aSgYjlA2Np/cbP31ioiIiGQqJcE05E2tqdF8ERERkUymoJ+GAiEtliUiIiKS6TS9ZhoKhiOcdsx4v8sQkTR24U8u9LsEEREZhIJ+mom0d3KouV0j+iKSUNXzqv0uQUREBqHWnTQTPDq1poK+iCTOzkd3svPRnX6XISIiA9CIfpr5x2JZCvoikjhrvr0GgOlLpvtciYiI9Ecj+mnm6GJZat0RERERyWgK+mmmu3WnSiP6IiIiIhlNQT/NBEKtFOZlU5SvriwRERGRTKagn2YC4RaqiwswM79LEREREREfadg3zQRCEarUny8iCfbOW9/pdwkiIjIIBf00Ewy3suC4CX6XISJprnxmud8liIjIINS6k0a6uhzBcEQX4opIwm29fytb79/qdxkiIjIAjeinkYYjbXR0OU2tKSIJt+6H6wCYefFMnysREZH+aEQ/jRydWlNBX0RERCTjKeinEa2KKyIiIiLdFPTTyH6tiisiIiIiUQr6aSQYipCdZVQU5ftdioiIiIj4LG2CvpndaGa7zCxiZs+a2bmD7J9nZt+MPqbVzPaY2ad67XOZmdVGt9ea2aWJfRUjEwhHqBiXT3aWFssSkcS69M5LufTOpP4vUUQk46VF0DezK4CbgO8CpwBrgQfNbNoAD7sbuBC4HpgJvA94sccxzwLuAVYA86Lff2dmCxLwEuJCU2uKyGgpmVpCydQSv8sQEZEBpMv0mp8Dljvnbo/e/qSZXQjcAHy5985m9jbgAuB451x99O5Xe+32GeBvzrnvRG9/x8zOj95/VVyrj5NAKML0irF+lyEiGeCle14CYO4Vc32uRERE+pPyI/pmlgecBjzSa9MjwMJ+HnYJsAH4nJntM7PtZvbfZjauxz5n9XHMhwc4pu8C4YguxBWRUbHx5xvZ+PONfpchIiIDSIcR/XIgGwj2uj8ILOnnMdOBc4BW4DKgFPgpMAl4b3Sf6n6OWd3XAc3serw2IKZNG6hjKDHaOrqYXV3MzOriUX9uEREREUk+6RD0hyMLcMDVzrkQgJl9AnjYzKqcc70D/qCcc7cBtwHMnz/fxbPYWOTlZLHyY2eN9tOKiIiISJJK+dYdoB7oBKp63V8FBPp5zH7gte6QH7U5+r17OD4wxGOKiIiIiCSNlA/6zrk24Flgaa9NS/Fm3+nLU8CkXj35NdHvu6Pf1w3xmCIiIiIiSSNdWnd+BNxpZs/ghfiP4fXb3wJgZncAOOfeH93/LuDfgP81s6/j9ejfBNzrnDsQ3ecmYI2ZfQn4I3ApcD5eb7+ISEa7/N7L/S5BREQGkRZB3zl3j5mVAV8FJgIvARc557pH56f12r/JzJbgXYC7ATiEF+a/1GOftWZ2JfBt4JvADuAK59zTCX45IiJJr7C80O8SRERkEObcqF83mvbmz5/vNm7UtHMikr42Ld8EwLxl83ytQ0REsP42pHyPvoiIjL5NyzcdDfsiIpKcFPRFRERERNKQgr6IiIiISBpS0BcRERERSUMK+iIiIiIiaSgtptcUEZHRdc1frvG7BBERGYSCvoiIDFluYa7fJYiIyCDUuiMiIkO24eYNbLh5g99liIjIABT0RURkyF5e+TIvr3zZ7zJERGQACvoiIiIiImlIQV9EREREJA0p6IuIiIiIpCEFfRERERGRNGTOOb9rSDtmVgfs9unpy4F6n547Xeg9HDm9hyOn93Dk9B7Gh97HkdN7OHJ6D/tX75y7sK8NCvppxsw2Oufm+11HKtN7OHJ6D0dO7+HI6T2MD72PI6f3cOT0Hg6PWndERERERNKQgr6IiIiISBpS0E8/t/ldQBrQezhyeg9HTu/hyOk9jA+9jyOn93Dk9B4Og3r0RURERETSkEb0RURERETSkIK+iIiIiEgaUtBPE2Z2o5ntMrOImT1rZuf6XVOqMLMvm9kGMwubWZ2Z3W9mc/2uK5VF31NnZv/jdy2pxswmmtmvoz+LETOrNbPFfteVKsws28y+1eP/w11m9m0zy/G7tmRlZovM7P/M7LXov9tlvbabmX3dzF43sxYze9zMTvSp3KQ00HtoZrlm9n0ze9HMjpjZfjO7y8ym+VhyUhrsZ7HXvrdG9/nCKJaYchT004CZXQHcBHwXOAVYCzyo/0Ridh5wM7AQeCvQATxqZhP8LCpVmdmZwPXAi37XkmrMrBR4CjDgn4DZwCeBAz6WlWr+Ffg48ClgFvDp6O0v+1lUkhsHvIT3XrX0sf1fgM/j/SyejvfzuMrMikatwuQ30HtYCJwKfCf6/d3AVOAhnYC+yWA/iwCY2XuBM4DXR6mulKWLcdOAmT0NvOic+0iP+7YD9zrn9MttiMxsHBACLnHO3e93PanEzEqA54APA18DXnLOfcLfqlKHmX0XWOycO9vvWlKVmT0ANDjnPtDjvl8DZc65d/pXWWowsybgE8655dHbhhem/sc5953ofWPwwv4XnHO3+lVrsur9HvazzxzgZeAk59zfR6u2VNLf+2hmx+ANaC4BHsT72fyv0a8wNWhEP8WZWR5wGvBIr02P4I1Qy9AV4f3bOOR3ISnoNrwTzL/5XUiKugR42szuMbMDZrbJzD4RDVsSmyeB881sFhwNVG8F/uJrVanrOKCaHr9jnHMtwBr0O2YkiqPf9XtmCKKfgPwW+LZzbrPf9aQCfWSU+sqBbCDY6/4g3tmuDN1NwCZgnc91pBQz+whwAnCt37WksOnAjcCPgf8A5gE/jW7T9Q6x+T7eyXqtmXXi/Z77jnPuZn/LSlnV0e99/Y6ZPMq1pIXoAN0Pgfudc/v8rifFfAOod8793O9CUoWCvkgPZvYj4BzgHOdcp9/1pAozm4l3jcg5zrl2v+tJYVnAxh4td8+b2Qy8HnMF/dhcAbwfuBqvNWIecJOZ7XLO/dLPwkSiI9K/AUqBd/lbTWoxs/OAZXj/piVGat1JffVAJ1DV6/4qIDD65aQuM/sxcBXwVufcTr/rSTFn4X269LKZdZhZB7AYuDF6O9/f8lLGfqC2132bAV1YH7sfAP/lnLvbOfd359ydwI/QxbjD1f17RL9jRqhH28lJwAXOuQafS0o15wETgf09fs8cA3zfzPTJSD8U9FOcc64NeBZY2mvTUryLVSQGZnYT/wj5W/yuJwX9EXgL3khL99dG4O7on9t8qSr1PAXM7HVfDbDbh1pSVSHe4EdPnej33XDtwgv0R3/HmFkBcC76HRMzM8sF7sEL+ec753SSNHQ3471/83p8vY7X6niBX0UlO7XupIcfAXea2TN4QeFjwCTgFl+rShFm9jPgOrwLIQ+ZWXdPapNzrsm3wlKIc+4wcLjnfWZ2BDjonHvJj5pS1I+BtWb2//BCwSl400R+xdeqUsv9wJfMbBde684pwOeAO3ytKolFZxo7IXozC5hmZvPw/v3uMbOfAF8xsy3ANuCrQBNwlw/lJqWB3kO8MPo7vKlJLwZcj98zoejFzcLgP4v0mmrYzNqBgHNu66gWmkI0vWaaMLMb8eY6nog3B+1nnXNr/K0qNZhZf/8IvuGc+/po1pJOzOxxNL3mkJnZP+Fd7zAT2IPXm/9Tp/+sYxKd2/1bwKVAJV471N3AN51zET9rS1bR3ue+Zsr6tXNuWXTWp68BHwXGA08DH9dJ/D8M9B4CX8f7ZKQvHxxoGs5MM9jPYh/7v4qm1xyQgr6IiIiISBpSz6KIiIiISBpS0BcRERERSUMK+iIiIiIiaUhBX0REREQkDSnoi4iIiIikIQV9EREREZE0pKAvIiIiIpKGFPRFRERERNKQgr6IiIiISBpS0BcRERERSUMK+iIiIiIiaUhBX0REREQkDSnoi4iIiIikIQV9EREREZE0pKAvIiIiIpKGFPRFRERERNKQgr6IiIiISBpS0BcRyUBmtszMnJmd4HctgzGzr5jZHjPrMLNNftczFGb2uJk97ncdIpKZcvwuQEREpD9mdgbwHeAHwB+BRl8LGrob/S5ARDKXgr6IiCSEmeU751pHeJjZ0e+3OOd2jrSm0dL92p1ztX7XIiKZS607IiKjwMy+Hm2VmWFmfzazJjPbbWb/bmZZPfbrbqk5tq/H97rPmdm3zezz0WM1R49dGf1aaWYhM9trZv/aT2mTzOyP0XoazOxnZjam1/MUmtn3zWyXmbVFv/+/XnWfF63nPWZ2u5nVAcFB3pMzzOzR6HMfMbPHoiP43dsfB5ZHb+6IHv/rAxyv0Mxujr6OJjP7PzM7J/q4ZT2P21c7jZm9ambLe913nJmtMLM6M2s1s01mdmmvfbr/buea2cNm1gSs7O+5zKzCzG4xs9eix9xiZtf32qfazH5tZq9H99lvZg+YWWX/76iIyBtpRF9EZHTdB/wv8GPgYuAbwN7ofcNxHfASXotIFfAT4A6gCHgQuA14H/AfZvZ359xfej3+N3ih9GbgDODfgbHAMgAzywEeBuYA3wL+DpwJ/BswAfh8r+P9NPq81wEF/RVtZicBq4Ha6HM54EvAajM70zn3QvQ1XQt8GXgPsB/YN8B7cStwBd57ugFYCtw1wP4DMrOpwNPAAeCzQF30+L83s0ucc//X6yF/An4JfB/o6ueYxcCTwBjg68Au4O3Az6OfAvw0uuudwDHAF/F+PqqAC4DC4b4eEck8CvoiIqPrh8657lD/qJm9FbiK4Qf9VuDdzrkOADObixdK/8059+3ofY8Dl+IF/t5B/y/OuS9E//xI9FODb5rZd51z26K1nQMsds6tie73mJkBfM3Mvu+cO9DjeM845z4cQ93/Hq39Aufc4Widq4BXga8B73HO1ZpZd7vO8865V/s7mJnNBK4G/p9z7j96vJ5xwMdiqKcvXwcM77U3RO97OHoC8E2gd9D/b+fcTYMc89N4Af4tzrnt0fseNbNSvPfz59G/y7OArzjnVvR47O+G+TpEJEOpdUdEZHT9udftl4BpIzjequ6QH7Ul+v3h7jui218Bpvbx+JW9bt+N97uhu4XmQmA3sNbMcrq/gEeAXLzR/Z7ui7HuRcAD3SE/WmcYLzwvjvEYPS2I1t3X6xmuC/FOjEK9XvvDwMnR0fmeYnntF+J9SrCrj2OW4X1yAt4nEl80s0+b2VssemYlIjIUGtEXERldB3vdbmWAFpcYHOp1u22A+/t6nt599N23J0e/V+KNQLf38/xlvW7v72e/3ib0s28AGB/jMXqaGP3e3+sZjkrg/dGvvpQB4R63Y3ntlcAJDP5+XoH3yca/4LVj7TezW4BvO+f6bAsSEelNQV9EJLlEot/zet3fO1DHSxXwcq/bAK9Fvzfg9ZFf3s/jX+112/W1Ux8OAtV93F/Nm09SYtEdsquAnrPzVPWxbwToPRoP3slHTw3AE3g99315vdftWF57A17P/6f72b4VINoO9XHg49G2pA/gXXtQB/w8hucREVHQFxFJMruj3+cC2+DoBbFvS9DzXQ78tcftK/EuJH06evsh4DKgyTm3hfhZDVxkZkXOuUYAMyvCu0D58WEc72m8ui8H/qPH/Vf2se9u4DIzy3POtUWfexHeBcw9PYTXK/+yc65lGDX15SHgk8CeXtc29Ms5txX4ipl9DO/nQkQkJgr6IiLJZQOwA/hBdPrKVrzZZ/IT9HwXmdkP8Hruz8BrF7mjx4WiK4AP4l2A+0PgBbxPG44H3gVc4pxrHsbzfgt4Z/S438cbDf9XvFllvjnUgznntprZXXgXEmfhvY9vAy7qY/e7geuBX0Wn0zwO+BwQ6rXfvwPPAGvM7H/wPr0Yjxe2pzvnPjTUOvFmW7oCeMLMfow3gj8WmAWc65x7t5mVAI/ivfdb8Np83h197keG8ZwikqEU9EVEkohzrsPM3g38DG8O+YN4PdpP44XweLsWb4rMG/D6+G8HumfhwTnXbmZvx5v68nq8UHwE72Tkz/zjmoAhcc69aGbn4a16+2u82W3W481w88IwX8tHgaZo/Xl4n1RcjTedZc/n/lt0dPwLeJ9WPI/3Pvy+1357zGw+3uw73wUq8FpvXorWPGTOuZCZLcQ7ifhXvGshDuMF/u7njwDPAR/Buz6iK7r9Gufcn4bzvCKSmcy5WNspRUREUot5C4/tAj7onFvubzUiIqNL02uKiIiIiKQhBX0RERERkTSk1h0RERERkTSkEX0RERERkTSkoC8iIiIikoYU9EVERERE0pCCvoiIiIhIGlLQFxERERFJQwr6IiIiIiJp6P8DTw2s3ahga5kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig = plt.figure(figsize=(12, 8))\n",
    "ax = plt.axes()\n",
    "\n",
    "data = np.vstack((np.arange(num_queries+1), np.array(results)))\n",
    "sns.lineplot(x=0, y=1, data=data)\n",
    "\n",
    "plt.xlabel('number of queries', labelpad=15)\n",
    "plt.ylabel('train accuracy', labelpad=25)\n",
    "\n",
    "earliest_stopping_response = np.amin([i for i, _ in enumerate(stopping_history) if stopping_history[i] is True])\n",
    "plt.axvline(x=earliest_stopping_response, ymin=0, ymax=1, color='purple', ls='--')\n",
    "\n",
    "sns.despine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4313da7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}