chapter-03-churn-prediction/04-metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train the model again first - to use its results later in this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction import DictVectorizer\n",
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('WA_Fn-UseC_-Telco-Customer-Churn.csv')\n",
    "\n",
    "df['TotalCharges'] = pd.to_numeric(df['TotalCharges'], errors='coerce')\n",
    "df['TotalCharges'] = df['TotalCharges'].fillna(0)\n",
    "\n",
    "df.columns = df.columns.str.lower().str.replace(' ', '_')\n",
    "\n",
    "string_columns = list(df.dtypes[df.dtypes == 'object'].index)\n",
    "\n",
    "for col in string_columns:\n",
    "    df[col] = df[col].str.lower().str.replace(' ', '_')\n",
    "\n",
    "df.churn = (df.churn == 'yes').astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_full, df_test = train_test_split(df, test_size=0.2, random_state=1)\n",
    "df_train, df_val = train_test_split(df_train_full, test_size=0.33, random_state=11)\n",
    "\n",
    "y_train = df_train.churn.values\n",
    "y_val = df_val.churn.values\n",
    "\n",
    "del df_train['churn']\n",
    "del df_val['churn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = ['gender', 'seniorcitizen', 'partner', 'dependents',\n",
    "               'phoneservice', 'multiplelines', 'internetservice',\n",
    "               'onlinesecurity', 'onlinebackup', 'deviceprotection',\n",
    "               'techsupport', 'streamingtv', 'streamingmovies',\n",
    "               'contract', 'paperlessbilling', 'paymentmethod']\n",
    "numerical = ['tenure', 'monthlycharges', 'totalcharges']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train_dict = df_train[categorical + numerical].to_dict(orient='records')\n",
    "\n",
    "dv = DictVectorizer(sparse=False)\n",
    "dv.fit(train_dict)\n",
    "\n",
    "X_train = dv.transform(train_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(random_state=1, solver='liblinear')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LogisticRegression(solver='liblinear', random_state=1)\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dict = df_val[categorical + numerical].to_dict(orient='records')\n",
    "X_val = dv.transform(val_dict)\n",
    "y_pred = model.predict_proba(X_val)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(random_state=1, solver='liblinear')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "small_subset = ['contract', 'tenure', 'totalcharges']\n",
    "train_dict_small = df_train[small_subset].to_dict(orient='records')\n",
    "dv_small = DictVectorizer(sparse=False)\n",
    "dv_small.fit(train_dict_small)\n",
    "\n",
    "X_small_train = dv_small.transform(train_dict_small)\n",
    "\n",
    "model_small = LogisticRegression(solver='liblinear', random_state=1)\n",
    "model_small.fit(X_small_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dict_small = df_val[small_subset].to_dict(orient='records')\n",
    "X_small_val = dv_small.transform(val_dict_small)\n",
    "\n",
    "y_pred_small = model_small.predict_proba(X_small_val)[:, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8016129032258065"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = model.predict_proba(X_val)[:, 1]\n",
    "churn = y_pred >= 0.5\n",
    "(churn == y_val).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8016129032258065"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_val, y_pred >= 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thresholds = np.linspace(0, 1, 11)\n",
    "thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.00 0.261\n",
      "0.05 0.501\n",
      "0.10 0.595\n",
      "0.15 0.640\n",
      "0.20 0.690\n",
      "0.25 0.730\n",
      "0.30 0.755\n",
      "0.35 0.767\n",
      "0.40 0.782\n",
      "0.45 0.795\n",
      "0.50 0.802\n",
      "0.55 0.790\n",
      "0.60 0.790\n",
      "0.65 0.788\n",
      "0.70 0.774\n",
      "0.75 0.752\n",
      "0.80 0.742\n",
      "0.85 0.739\n",
      "0.90 0.739\n",
      "0.95 0.739\n",
      "1.00 0.739\n"
     ]
    }
   ],
   "source": [
    "thresholds = np.linspace(0, 1, 21)\n",
    "\n",
    "accuracies = []\n",
    "\n",
    "for t in thresholds:\n",
    "    acc = accuracy_score(y_val, y_pred >= t)\n",
    "    accuracies.append(acc)\n",
    "    print('%0.2f %0.3f' % (t, acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAqzklEQVR4nO3deZwU1bn/8c/DsCOICCoiIgkIARdUQkSRRUQWQXZ1MHrVG/kRd02M5iY3GtcYFSUaJWiM8RpAURAEZAkqS0QQE3SGRUSURVDZZF+H5/dH1WA79kAjU93TU9/369Uvajld56mmp56uOlXnmLsjIiLxVS7TAYiISGYpEYiIxJwSgYhIzCkRiIjEnBKBiEjMKRGIiMScEoFEyszuNrMX01DPVWY263u+94AxmtlnZnbB949OpHRTIpDDYmZbE177zGxHwvzlmY4vm5hZw/AzfCrTsUi8KBHIYXH3IwpfwAqgR8KyfxzKtsysfDRRZo0rgY3AZWZWKZ0Vm1lOOuuT0kWJQNKhopm9YGZbzGyBmbUsXBFedrnDzD4EtplZeTM728zeMbOvzewDM2ufUP4qM1sWbuvTomcdZvaImW0M13VNWH68mY0zsw1mttTMri0uWDO7wsyWm9l6M/vNAcqdbWZfJB5Ezax3uC+YWSszm2dmm83sSzMbfJDP6Urgt8AeoEeRunqa2fxwW5+YWZdweS0z+5uZrQ73+7WEz2lWkW24mTUKp583s6fNbKKZbQM6mNlFZvafsI6VZnZ3kfe3Sfh/WRnW8eNw38onlOtrZvMPsq9Smri7XnqVyAv4DLigyLK7gZ1ANyAHeBB4t8h75gP1gSpAPWB9WL4c0CmcrwNUAzYDTcL31gWah9NXERxArw3r+TmwGrBw/XTgKaAy0AJYC3RMiPHFcLoZsBVoC1QCBgN7i+5XQvyfAJ0S5kcBd4bTs4ErwukjgLMP8NmdB+wCjgKeAMYlrGsFbAo/i3LhZ9Q0XDcBeCl8XwWgXcLnMatIHQ40CqefD7d5brjNykB74NRw/jTgS6BXWP5EYAuQG9ZzNNAiXLcQ6JpQzxjgF5n+PuqV+ktnBJIOs9x9orsXAP8HnF5k/Z/cfaW77wB+CkwMy+9z96nAPILEALAPOMXMqrj7GndfkLCd5e7+TFjP3wkSxbFmVh9oA9zh7jvdfT7wLHBFklj7AePdfYa77wL+N6yzOCMIDo6YWfUwzhHhuj1AIzOr7e5b3f3dA2znv4A33H0jMBzoambHhOv+G3jO3aeGn8nn7r7YzOoCXYFB7r7R3fe4+/QD1FHUWHf/V7jNne7+trvnhfMfhvvRLix7OfBPdx8R1rM+/Bwh+Kx/Gn4GtYDO4T5IllAikHT4ImF6O1C5SHvAyoTpBkD/8PLD12b2NcFBvK67bwMuBQYBa8xsgpk1TVaPu28PJ48Ajgc2uPuWhLLLCX5ZF3V8YjxhnesPsG/DgT7hNf0+wL/dfXm47r+Bk4HFZvaemXVPtgEzqwL0B/4R1jmboL1lQFikPsGZR1H1w/3aeID4DiTxc8fMfmJmb5nZWjPbRPA51z5IDAAvAj3M7AjgEmCmu6/5njFJBigRSGmQ2AXuSuD/3L1mwquau/8BwN0nu3sngl/7i4FnUtj+aqBW+Iu90InA50nKriE46AFgZlUJLoMkD9x9IUFS6Upw4B6esO5jd88FjgEeAl4xs2pJNtMbqAE8FbY5fEGQpK4M168EfpjkfSvD/aqZZN02oGrCfhyXLPwi88OBcUB9dz8SGArYQWLA3T8nuAzWm+As6/+SlZPSS4lASpvCX5edzSzHzCqbWXszO8HMjjWzi8OD6S6Ca/kFB9ugu68E3gEeDLd3GsGv9WR3Nb0CdA8bRisC93Dwv5PhwE0E7QqjChea2U/NrI677wO+Dhcni/e/gOcIrs+3CF/nAi3M7FTgr8DVZtbRzMqZWT0zaxr+6n6DIIEcZWYVzKxtuM0PgOZm1sLMKhO0gxxMdYIzjJ1m1opvzkgg+KwuMLNLLGjQP9rMWiSsfwH4VbgPY1KoS0oRJQIpVcKDdk/gfwgadFcCtxN8V8sBvyD4hb+B4Pr1dSluOhc4KXzvGOCusP2haP0LgOsJDu5rCG7nXHWQbY8gaGh9093XJSzvAiwws63AEOAyd9+Z+EYzqwd0BB539y8SXu8Dk4D/cve5wNXAYwQNvNMJLqFB8At8D8HZ0VfALeF+LCFIYv8EPgZSedjuOuAeM9sC/A54OeFzWUHQ/vELgs9+Pt9u6xkTxjQmvJwmWaTwjgoRkcNiZp8A/8/d/5npWOTQ6IxARA6bmfUlaHN4M9OxyKGL+5OcInKYzOxtgucvrgjbQyTL6NKQiEjM6dKQiEjMZd2lodq1a/tJJ52U6TBERLLK+++/v87d6yRbl3WJ4KSTTmLevHmZDkNEJKuY2fLi1unSkIhIzCkRiIjEnBKBiEjMKRGIiMScEoGISMwpEYiIxFykicDMupjZR+EYsXcmWX+kmb1uwbi0C8zs6ijjERGR74rsOYJwQO8/E4yzugp4z8zGhQN5FLoeWOjuPcysDvCRmf3D3XdHFZdISdq2bRtLlixh8eLFLF++nFNPPZV27dpxxBFHZDo0kZRF+UBZK2Cpuy8DMLORBP3MJyYCB6qbmREMKbiBYKBwkVLD3Vm9ejWLFy9m8eLFfPTRR/unV65c+Z3yFSpU4JxzzuHCCy+kU6dOnHnmmeTk5GQgcpHURNbpnJn1A7q4+8/C+SuAn7j7DQllqhMMjdeUYHSkS919QpJtDQQGApx44olnLV9e7ANyIt/bvn37WLhwIYsWLfrWwf6jjz5i69at+8tVr16dJk2a0LRp0/2vJk2aUL9+febNm8eUKVOYOnUq//nPfwCoVasWHTt23J8YGjRoUFwIIpExs/fdvWXSdREmgv5A5yKJoJW735hQph/BkHy3EYyHOhU43d03F7fdli1burqYkJKyd+9eZsyYwejRoxkzZgyrV6/ev+7EE0/8zsG+adOm1K1bl+Ak9sC++uorpk2btj8xfP55METyySefTKdOnbjwwgtp3749NWrUiGz/RAplKhG0Bu52987h/K8B3P3BhDITgD+4+8xw/k3gznBovqSUCORw7dq1i2nTpjF69GjGjh3LunXrqFKlCl27duXiiy/m9NNPp3HjxlSrlmyc+e/H3Vm0aBFTp05lypQpvP3222zfvp2cnBxat269/0whJyeH8uXLk5OT853pA62rVKkSVatW3f+qUqWKLkfJt2QqEZQHlhCMx/o58B4wIBwTtrDM08CX7n63mR0L/JvgjGBdsm2CEoF8P9u3b2fSpEmMHj2a119/nc2bN1O9enV69OhB37596dy5c4ke+A9m165dzJ49e39ieP/99ynpv8WKFSt+KzkU96pbty59+/blzDPPTOlMR7JTRhJBWHE34HEgB3jO3e83s0EA7j7UzI4HngfqAkZwdvDigbapRCCp2rx5M+PHj2f06NG88cYbbN++naOPPpqePXvSt29fOnbsSKVKlTIdJgAbN25k48aNFBQUsHfvXgoKCr4zfaB1u3btYseOHWzfvv2QXjt27GDNmjXs2bOHxo0bk5ubS25uLk2bNs30RyIlLGOJIApKBHIgGzZsYOzYsbz66qtMnTqV3bt3U7duXXr37k3fvn1p27Yt5ctnXe/rkdqwYQOjR49mxIgRvPXWW7g7Z5xxBrm5uVx22WXUr18/0yFKCVAikDJt27ZtjBs3juHDhzN58mT27NlDgwYN6Nu3L3369KF169aUK6eH6FOxevVqXn75ZUaMGMHcuUFT3XnnnUdubi79+vWjTp2k45pIFlAikDJn9+7dTJ48meHDhzNu3Di2b99OvXr1yM3N5dJLL+Wss87S9e7DtHTpUkaOHMmIESNYuHAhOTk5XHjhheTm5tKrVy+qV6+e6RDlECgRSJlQUFDAjBkzGDFiBK+88gobN27k6KOPpn///uTm5tKmTRv98o+Au5OXl8fw4cMZOXIky5cvp3LlynTv3p3LL7+cHj166A6lLKBEIFnL3Zk3bx4jRoxg5MiRrFmzhmrVqtGrVy8GDBhAp06dqFChQqbDjA13Z/bs2YwYMYKXXnqJtWvX0qxZM+655x569+6tRFyKKRFI1lm0aNH+X6BLly6lYsWKdO3alQEDBtC9e3eqVq2a6RBjb+/evYwZM4a77rqLRYsWccYZZ3DvvffSrVs3XZYrhZQIpNQrKChgzpw5jB8/ntdff538/HzKlStHhw4dyM3NpU+fPhx11FGZDlOSKCgoYMSIEdx1110sW7aM1q1bc99993H++ednOjRJoEQgpdKmTZuYPHky48ePZ+LEiaxfv56cnBzOO+88evXqxSWXXELdunUzHaakaM+ePTz//PPcc889rFq1ig4dOnDfffdxzjnnZDo0QYlASpGPP/6Y119/nfHjxzNz5kz27t1LrVq16NatG927d6dz587UrFkz02HKYdi5cyfDhg3jgQce4Msvv6Rbt27ce++9nHnmmZkOLdaUCCRj9uzZw6xZsxg/fjzjx49nyZIlADRv3pwePXrQvXt3zj77bN11UgZt27aNJ598koceeoiNGzfSt29ffv/739O8efNMhxZLSgSSVvv27ePVV1/llVdeYdKkSWzevJmKFSvSoUMHunfvzkUXXUTDhg0zHaakyaZNm3j88cd59NFH2bp1KwMGDODuu++mUaNGmQ4tVpQIJG3mz5/Pddddx+zZsznuuOO46KKL6N69OxdccIFG7Yq59evX8/DDD/OnP/2J3bt3c/XVV3PllVfSqlWrUtPnU1mmRCCR27x5M7/73e944oknOProo3n44Ye54oordF+5fMcXX3zBgw8+yNChQ9m9ezeVK1fm7LPPpl27drRr146zzz6bKlWqZDrMMkeJQCLj7owcOZLbbruNL7/8kkGDBnH//ffrVk85qA0bNjBz5kymT5/O9OnTmT9/Pvv27aNChQq0atVqf2I455xzdDZZApQIJBKLFi3i+uuv56233qJly5Y8/fTTtGyZ9HsmclCbNm1i1qxZTJ8+nRkzZjBv3jwKCgooX748Z511Fu3ataNt27a0adOGI488MtPhZh0lAilR27Zt4/777+eRRx6hWrVqPPjgg1x77bW680dK1NatW3nnnXf2nzHMnTuXPXv2UK5cOVq0aEHz5s2pXbv2/tfRRx/9rflatWqp+5EESgRSItydsWPHcvPNN7NixQquuuoqHnroIY455phMhyYxsH37dt599939ZwyfffYZ69atY+vWrcW+p2bNmkmTRfXq1bOyG4xzzz2Xjh07fq/3HigRaIQOScmyZcu46aabmDBhAqeccgozZ86kTZs2mQ5LYqRq1aqcf/753+m6YufOnaxfv55169Z961V02eeff84HH3zA2rVr2blzZ4b24vDccccd3zsRHIgSgRzQzp07efjhh3nggQcoX748jz76KDfeeKNOuaXUqFy5MvXq1aNevXopv2ffvn0RRpR9lAikWFOmTOH6669n6dKlXHLJJQwePPiQ/thESivd1vxt+jQkqQcffJDOnTtjZkyZMoWXXnpJSUCkjNIZgXyLu/Pb3/6WBx54gMsvv5y//vWveupTpIxTIpD93J1bb72VIUOGMHDgQJ5++mmdQovEgP7KBQgGFxk4cCBDhgzhlltuYejQoUoCIjGhv3Rh7969XHnllTz77LP85je/YfDgwVl5j7WIfD+6NBRzu3btIjc3lzFjxvDAAw/w61//OtMhiUiaKRHE2I4dO+jTpw+TJk1iyJAh3HTTTZkOSUQyQIkgprZs2cLFF1/M9OnTeeaZZ/jZz36W6ZBEJEMibSMwsy5m9pGZLTWzO5Osv93M5oevfDMrMLNaUcYk8PXXX3PhhRcyc+ZMXnzxRSUBkZiLLBGYWQ7wZ6Ar0AzINbNmiWXc/WF3b+HuLYBfA9PdfUNUMQmsXbuWDh068P777zNq1CgGDBiQ6ZBEJMOiPCNoBSx192XuvhsYCfQ8QPlcYESE8cTemjVraN++PYsXL2bcuHH07t070yGJSCkQZSKoB6xMmF8VLvsOM6sKdAFeLWb9QDObZ2bz1q5dW+KBxsHy5cs577zzWLFiBZMmTaJLly6ZDklESokoE0GyG9GLG/ygB/Cv4i4Lufswd2/p7i3r1KlTYgHGxccff8x5553H+vXrmTp1Ku3atct0SCJSikR519AqoH7C/AnA6mLKXoYuC0ViwYIFXHDBBezdu5c333yTM844I9MhiUgpE+UZwXtAYzNraGYVCQ7244oWMrMjgXbA2AhjiaUPP/yQdu3aYWZMnz5dSUBEkorsjMDd95rZDcBkIAd4zt0XmNmgcP3QsGhvYIq7b4sqljhas2YNF110EZUrV+btt9+mUaNGmQ5JREqpSB8oc/eJwMQiy4YWmX8eeD7KOOJmx44d9OzZk40bNzJr1iwlARE5ID1ZXMbs27ePq666innz5jFmzBhatGiR6ZBEpJRTIihjfv/73/Pyyy/zxz/+kZ49D/TYhohIQN1QlyHDhw/nnnvu4eqrr+aXv/xlpsMRkSyhRFBGzJ49m2uuuYa2bdsydOhQjScgIilTIigDli9fTq9evTjhhBN49dVXqVixYqZDEpEsojaCLLdlyxZ69OjBrl27mD59OrVr1850SCKSZZQIslhBQQG5ubksXLiQN954g6ZNm2Y6JBHJQkoEWexXv/oVEyZM4KmnnqJTp06ZDkdEspTaCLLUM888w+DBg7nxxhv5+c9/nulwRCSLKRFkobfeeovrrruOzp07M3jw4EyHIyJZTokgyyxZsoS+ffty8skn89JLL1G+vK7uicjhUSLIIhs3bqRHjx7k5OTw+uuvc+SRR2Y6JBEpA/RzMkvs2bOHfv368dlnnzFt2jR+8IMfZDokESkjlAiygLtzww038Oabb/L3v/+dNm3aZDokESlDdGkoCwwZMoRhw4bx61//miuvvDLT4YhIGaNEUMpNmDCBX/ziF/Tp04f77rsv0+GISBmkRFCKzZkzh0suuYQWLVrwwgsvUK6c/rtEpOTpyFJKLVq0iG7dulG3bl0mTpxItWrVMh2SiJRRSgSl0MqVK+ncuTMVKlRgypQpHHvssZkOSUTKMN01VMps2LCBLl26sGnTJqZPn67bREUkckoEpcj27dvp3r07n3zyCZMmTdJ4wyKSFkoEpcSePXvo378/c+bMYdSoUbRv3z7TIYlITCgRlAL79u3jZz/7GRMnTuQvf/kLffr0yXRIIhIjaiwuBe644w5eeOEF7rnnHgYOHJjpcEQkZpQIMuyRRx7hkUce4frrr+e3v/1tpsMRkRhSIsigF154gdtvv51LLrmEIUOGYGaZDklEYkiJIEMmTJjANddcQ8eOHXnhhRfIycnJdEgiElNKBBnwzjvv0L9/f1q0aMGYMWOoVKlSpkMSkRiLNBGYWRcz+8jMlprZncWUaW9m881sgZlNjzKe0mDBggV0796dE044gYkTJ1K9evVMhyQiMRfZ7aNmlgP8GegErALeM7Nx7r4woUxN4Cmgi7uvMLNjooqnNFixYgWdO3emUqVKTJ48mWOOKdO7KyJZIsozglbAUndf5u67gZFAzyJlBgCj3X0FgLt/FWE8GbVu3To6d+7M1q1bmTx5Mg0bNsx0SCIiQLSJoB6wMmF+Vbgs0cnAUWb2tpm9b2ZJR10xs4FmNs/M5q1duzaicKOzbds2unfvzqeffsq4ceM47bTTMh2SiMh+UT5ZnOxeSE9S/1lAR6AKMNvM3nX3Jd96k/swYBhAy5Yti26j1Lv88st57733GD16NG3bts10OCIi3xJlIlgF1E+YPwFYnaTMOnffBmwzsxnA6cASyoi5c+cyduxY7r//fnr2LHplTEQk86K8NPQe0NjMGppZReAyYFyRMmOB88ysvJlVBX4CLIowprR77LHHqFGjBjfccEOmQxERSSqyMwJ332tmNwCTgRzgOXdfYGaDwvVD3X2RmU0CPgT2Ac+6e35UMaXbypUrGTVqFDfffDM1atTIdDgiIkkdNBGYWXdgorvvO9SNu/tEYGKRZUOLzD8MPHyo284GTzzxBO7OTTfdlOlQRESKlcqlocuAj83sj2b2o6gDKiu2bNnCsGHD6NevHw0aNMh0OCIixTpoInD3nwJnAJ8AfzOz2eHtnHok9gD+9re/sWnTJm677bZMhyIickApNRa7+2bgVYKHwuoCvYF/m9mNEcaWtQoKCnj88cc555xz+MlPfpLpcEREDuigicDMepjZGOBNoALQyt27Etzm+cuI48tKY8eO5dNPP9XZgIhkhVTuGuoPPObuMxIXuvt2M7smmrCy22OPPcZJJ51Er169Mh2KiMhBpXJp6C5gbuGMmVUxs5MA3H1aRHFlrblz5zJr1ixuvvlmjTEgIlkhlUQwiuAe/0IF4TJJovABsmuu0cmSiGSHVBJB+bD3UADC6YrRhZS9VqxYwahRo7j22mv1AJmIZI1UEsFaM7u4cMbMegLrogspez3xxBMA3HijbqYSkeyRSmPxIOAfZvYkQY+iK4Gk3UXHmR4gE5FsddBE4O6fAGeb2RGAufuW6MPKPs899xybN2/m1ltvzXQoIiKHJKVO58zsIqA5UNksGGbA3e+JMK6sUlBQwJAhQ/QAmYhkpVQeKBsKXArcSHBpqD+gax8J9ACZiGSzVBqLz3H3K4GN7v57oDXfHnAm9gYPHkzDhg31AJmIZKVUEsHO8N/tZnY8sAfQyOuhOXPm8K9//UsPkIlI1kqljeB1M6tJMGbAvwnGHX4myqCyiR4gE5Fsd8BEYGblgGnu/jXwqpmNByq7+6Z0BFfarVixgldeeYVbbrmF6tXVK7eIZKcDXhoKRyV7NGF+l5LAN/QAmYiUBam0EUwxs75WeN+oAHqATETKjlTaCG4DqgF7zWwnwS2k7u6x7kyn8AEy3TIqItkulSeLdfG7iMIRyM4991xatWqV6XBERA7LQROBmbVNtrzoQDVx8tprr/HZZ5/x6KOPHrywiEgpl8qlodsTpisDrYD3gfMjiSgLFD5A1rNnz0yHIiJy2FK5NNQjcd7M6gN/jCyiUm7OnDm88847PP7443qATETKhFTuGipqFXBKSQeSLfQAmYiUNam0ETxB8DQxBImjBfBBhDGVWsuXL+eVV17h1ltv1QNkIlJmpNJGMC9hei8wwt3/FVE8pZoeIBORsiiVRPAKsNPdCwDMLMfMqrr79oO90cy6AEOAHOBZd/9DkfXtgbHAp+Gi0aV1nIPNmzfzzDPP0L9/f0488cRMhyMiUmJSaSOYBlRJmK8C/PNgbzKzHODPQFegGZBrZs2SFJ3p7i3CV6lMAqARyESk7EolEVR2962FM+F01RTe1wpY6u7L3H03MBLIyvstC0cg0wNkIlIWpZIItpnZmYUzZnYWsCOF99UjGOi+0KpwWVGtzewDM3vDzJqnsN20W7BgAZ999hkDBw7MdCgiIiUulTaCW4BRZrY6nK9LMHTlwSTrpM6LzP8baODuW82sG/Aa0Pg7GzIbCAwEMnJ9Pj8/H4Czzjor7XWLiEQtlQfK3jOzpkATgoP7Ynffk8K2V/HtIS1PAFYnFnD3zQnTE83sKTOr7e7ripQbBgwDaNmyZdFkErm8vDwqVKhA48bfyVEiIlkvlcHrrwequXu+u+cBR5jZdSls+z2gsZk1NLOKwGXAuCLbPq6we2szaxXGs/5QdyJq+fn5NGnShIoVK2Y6FBGREpdKG8G14QhlALj7RuDag73J3fcCNwCTgUXAy+6+wMwGmdmgsFg/IN/MPgD+BFzm7mn/xX8weXl5nHrqqZkOQ0QkEqm0EZQzMys8QIe3hab009jdJwITiywbmjD9JPBk6uGm3+bNm1m+fLkaikWkzEolEUwGXjazoQSNvYOANyKNqhRZuHAhgM4IRKTMSiUR3EFwx87PCRqL/0Nw51As5OXlAXDKKbHtZ09EyriDthGEA9i/CywDWgIdCa75x0J+fj7VqlXTuMQiUmYVe0ZgZicT3OmTS3Anz0sA7t4hPaGVDnl5eZxyyimUK/d9euwWESn9DnR0W0zw67+Hu7dx9yeAgvSEVXrk5+frspCIlGkHSgR9gS+At8zsGTPrSPKnhcusL7/8krVr16qhWETKtGITgbuPcfdLgabA28CtwLFm9rSZXZim+DKqsGsJnRGISFmWSmPxNnf/h7t3J+gmYj5wZ9SBlQaFiUBnBCJSlh1SC6i7b3D3v7j7+VEFVJrk5eVRp04djjnmmEyHIiISGd0KcwBqKBaROFAiKMa+ffvIz8/XZSERKfOUCIqxfPlytm3bpjMCESnzlAiKoYZiEYkLJYJiFPYx1KxZswxHIiISLSWCYuTn59OgQQNq1KiR6VBERCKlRFAMDUYjInGhRJDE7t27Wbx4sRqKRSQWlAiSWLJkCXv37tUZgYjEghJBEupjSETiRIkgifz8fHJycmjSpEmmQxERiZwSQRJ5eXk0adKESpUqZToUEZHIKREkoT6GRCROlAiK2Lp1K8uWLVNDsYjEhhJBEQsXLgTUUCwi8aFEUIT6GBKRuFEiKCIvL48qVarQsGHDTIciIpIWSgRF5Ofn07x5c8qV00cjIvGgo10R6mNIROIm0kRgZl3M7CMzW2pmxQ54b2Y/NrMCM+sXZTwHs3btWr788ks1FItIrESWCMwsB/gz0BVoBuSa2Xc69w/LPQRMjiqWVC1YsABQQ7GIxEuUZwStgKXuvszddwMjgZ5Jyt0IvAp8FWEsKSkcjEZnBCISJ1EmgnrAyoT5VeGy/cysHtAbGHqgDZnZQDObZ2bz1q5dW+KBFsrPz6dWrVocd9xxkdUhIlLaRJkILMkyLzL/OHCHuxccaEPuPszdW7p7yzp16pRUfN9R2FBslix0EZGyKcpEsAqonzB/ArC6SJmWwEgz+wzoBzxlZr0ijKlY7q4+hkQklspHuO33gMZm1hD4HLgMGJBYwN33P7VlZs8D4939tQhjKtaKFSvYsmWLGopFJHYiSwTuvtfMbiC4GygHeM7dF5jZoHD9AdsF0k2D0YhIXEV5RoC7TwQmFlmWNAG4+1VRxnIwSgQiEld6sjiUl5dH/fr1OfLIIzMdiohIWikRhNRQLCJxpUQA7Nmzh0WLFqmhWERiSYkAWLp0Kbt379YZgYjEkhIBGoxGROJNiYCgobhcuXI0bdo006GIiKSdEgHBGUHjxo2pXLlypkMREUk7JQI0GI2IxFvsE8H27dv55JNP1FAsIrEV+0SwcOFC3F1nBCISW7FPBOpaQkTiTokgP5/KlSvzwx/+MNOhiIhkROwTQV5eHs2aNSMnJyfToYiIZETsE4H6GBKRuIt1ItiwYQOrV69WQ7GIxFqsE4EaikVElAgA9TEkIvEW60SQl5dHzZo1Of744zMdiohIxsQ6ERQ2FJtZpkMREcmY2CYCd1cfQyIixDgRfP7552zatEkNxSISe7FNBHl5eYAaikVEYpsICu8Yat68eYYjERHJrFgnguOPP55atWplOhQRkYyKbSJQQ7GISCCWiaCgoICFCxeqoVhEhJgmgqVLl7Jr1y6dEYiIENNEoD6GRES+EWkiMLMuZvaRmS01szuTrO9pZh+a2Xwzm2dmbaKMp1B+fj5mRrNmzdJRnYhIqVY+qg2bWQ7wZ6ATsAp4z8zGufvChGLTgHHu7mZ2GvAy0DSqmArl5eXRqFEjqlSpEnVVIiKlXpRnBK2Ape6+zN13AyOBnokF3H2ru3s4Ww1w0kCD0YiIfCPKRFAPWJkwvypc9i1m1tvMFgMTgGuSbcjMBoaXjuatXbv2sILasWMHH3/8sRqKRURCUSaCZF16fucXv7uPcfemQC/g3mQbcvdh7t7S3VvWqVPnsIJavHgx+/bt0xmBiEgoykSwCqifMH8CsLq4wu4+A/ihmdWOMCb1MSQiUkSUieA9oLGZNTSzisBlwLjEAmbWyMLBAMzsTKAisD7CmMjPz6dixYo0atQoympERLJGZHcNufteM7sBmAzkAM+5+wIzGxSuHwr0Ba40sz3ADuDShMbjSOTn5/OjH/2I8uUj23URkawS6dHQ3ScCE4ssG5ow/RDwUJQxFJWXl0f79u3TWaWISKkWqyeLv/76a1atWqWGYhGRBLFKBIVdS6ihWETkG7FMBDojEBH5RuwSQY0aNahfv/7BC4uIxESsEkFeXh6nnHIK4R2rIiJCjBKBu6uPIRGRJGKTCNasWcOGDRvUUCwiUkRsEoEaikVEkotNIqhatSoXX3yxzghERIqITT8Lbdq0oU2btAyAJiKSVWJzRiAiIskpEYiIxJwSgYhIzCkRiIjEnBKBiEjMKRGIiMScEoGISMwpEYiIxJxFPERwiTOztcDy7/n22sC6EgwnG+rWPsej7rjVm8m6s3WfG7h7nWQrsi4RHA4zm+fuLeNUt/Y5HnXHrd5M1l0W91mXhkREYk6JQEQk5uKWCIbFsG7tczzqjlu9may7zO1zrNoIRETku+J2RiAiIkUoEYiIxFyZTARm1sXMPjKzpWZ2Z5L1ZmZ/Ctd/aGZnpqnepmY228x2mdkvS6LOQ6j78nBfPzSzd8zs9DTV2zOsc76ZzTOzEhsd6GB1J5T7sZkVmFm/dNRrZu3NbFO4z/PN7HclUW8qdSfUP9/MFpjZ9HTUa2a3J+xvfvh510pT3Uea2etm9kG4z1enqd6jzGxM+P2ea2YlMg6umT1nZl+ZWX4x60v++OXuZeoF5ACfAD8AKgIfAM2KlOkGvAEYcDYwJ031HgP8GLgf+GWa9/kc4Khwumsa9/kIvmmLOg1YnK59Tij3JjAR6JemfW4PjM/Qd7smsBA4sfA7l67POqF8D+DNNO7z/wAPhdN1gA1AxTTU+zBwVzjdFJhWQvvcFjgTyC9mfYkfv8riGUErYKm7L3P33cBIoGeRMj2BFzzwLlDTzOpGXa+7f+Xu7wF7DrOu71P3O+6+MZx9FzghTfVu9fDbC1QDSuruhFT+nwFuBF4FvkpzvVFIpe4BwGh3XwHBdy5N9SbKBUaUQL2p1u1AdTMzgh8eG4C9aai3GTANwN0XAyeZ2bGHWS/uPoNgH4pT4sevspgI6gErE+ZXhcsOtUwU9UblUOv+b4JfFGmp18x6m9liYAJwTQnUm1LdZlYP6A0MLaE6U6o31Dq8VPGGmTVPY90nA0eZ2dtm9r6ZXZmmegEws6pAF4LkWxJSqftJ4EfAaiAPuNnd96Wh3g+APgBm1gpoQMn8wCqJ2A5JWUwElmRZ0V+hqZSJot6opFy3mXUgSAR3pKtedx/j7k2BXsC9JVBvqnU/Dtzh7gUlVGeq9f6boF+X04EngNfSWHd54CzgIqAz8L9mdnIa6i3UA/iXux/oF21J190ZmA8cD7QAnjSzGmmo9w8ESXc+wZnnfzj8M5FUlPixpvzhvLmUWgXUT5g/geCXwqGWiaLeqKRUt5mdBjwLdHX39emqt5C7zzCzH5pZbXc/3E67Uqm7JTAyuGJAbaCbme1199eirNfdNydMTzSzp9K4z6uAde6+DdhmZjOA04ElEddb6DJK7rJQqnVfDfwhvAS51Mw+JbhmPzfKesP/56shaMAFPg1fUSv5Y01JNG6UphdBclsGNOSbRp7mRcpcxLcbW+amo96EsndTso3FqezzicBS4Jw019uIbxqLzwQ+L5xP1+cdln+ekmksTmWfj0vY51bAinTtM8Elkmlh2apAPnBKOj5r4EiCa9vV0vwdexq4O5w+NvyO1U5DvTUJG6WBawmu25fUfp9E8Y3FJX/8KqnAS9OLoFV9CUGr/2/CZYOAQeG0AX8O1+cBLdNU73EE2Xwz8HU4XSNNdT8LbCQ4hZ4PzEtTvXcAC8I6ZwNt0vX/XKTs85RAIkhxn28I9/kDgob5kky+B91n4HaCO4fygVvSWO9VwMiS2tdD+LyPB6aEf8v5wE/TVG9r4GNgMTCa8K68Eqh3BLCG4KaSVQSXciM9fqmLCRGRmCuLjcUiInIIlAhERGJOiUBEJOaUCEREYk6JQEQk5pQIJDbM7OiEHjK/MLPPw+mvzWxhBPXdbYfYy6yZbS1m+fMl1XuqSFFKBBIb7r7e3Vu4ewuC/oceC6dbAAftm8bMyuKT+CJKBCKhHDN7JuzPfoqZVQEIO297IOzX/2YzO8vMpocduk0u7PXRzG4ys4Vh//AjE7bbLNzGMjO7qXChmd0W9tufb2a3FA0m7HP+yXCbEwi6MBeJhH7hiAQaA7nufq2ZvQz0BV4M19V093ZmVgGYDvR097VmdinB2BLXAHcCDd19l5nVTNhuU6ADUB34yMyeJhiX4WrgJwRPic4xs+nu/p+E9/UGmgCnEnSbsBB4LoodF1EiEAl86u7zw+n3Cfp6KfRS+G8T4BRgatiRXQ5BVwAAHwL/MLPX+HZvoxPcfRewy8y+IjiotwHGeNAxHGY2GjiPoPfKQm2BER70nLrazN48/F0USU6JQCSwK2G6AKiSML8t/NeABe7eOsn7LyI4eF9M0PVz4RgERbdbnuTdCCej/l8kLdRGIJK6j4A6ZtYawMwqmFlzMysH1Hf3t4BfEfRKecQBtjMD6GVmVc2sGsFloJlJylxmZjlhO0SHEt4Xkf10RiCSInffHd7C+SczO5Lg7+dxgh4qXwyXGcHdSF+Hl4+SbeffZvY83/SX/2yR9gGAMcD5BL1LLiFomxCJhHofFRGJOV0aEhGJOSUCEZGYUyIQEYk5JQIRkZhTIhARiTklAhGRmFMiEBGJuf8PFBFKYfcMiyYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(thresholds, accuracies, color='black')\n",
    "\n",
    "plt.title('Threshold vs Accuracy')\n",
    "plt.xlabel('Threshold')\n",
    "plt.ylabel('Accuracy')\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "\n",
    "# plt.savefig('04_threshold_accuracy.svg')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7672043010752688"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "churn_small = y_pred_small >= 0.5\n",
    "(churn_small == y_val).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7672043010752688"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_val, churn_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size_val = len(y_val)\n",
    "baseline = np.repeat(False, size_val)\n",
    "baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7387096774193549"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(baseline, y_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_positive = ((y_pred >= 0.5) & (y_val == 1)).sum()\n",
    "false_positive = ((y_pred >= 0.5) & (y_val == 0)).sum()\n",
    "false_negative = ((y_pred < 0.5) & (y_val == 1)).sum()\n",
    "true_negative = ((y_pred < 0.5) & (y_val == 0)).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1202,  172],\n",
       "       [ 197,  289]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table = np.array(\n",
    "     # predict neg    pos\n",
    "    [[true_negative, false_positive], # actual neg\n",
    "     [false_negative, true_positive]]) # actual pos\n",
    "\n",
    "confusion_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.64623656, 0.09247312],\n",
       "       [0.10591398, 0.15537634]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table / confusion_table.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision and recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6268980477223427, 0.5946502057613169)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = true_positive / (true_positive + false_positive)\n",
    "recall = true_positive / (true_positive + false_negative)\n",
    "precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.64623656, 0.09247312],\n",
       "       [0.10591398, 0.15537634]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_table / confusion_table.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6268980477223427, 0.5946502057613169)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision = true_positive / (true_positive + false_positive)\n",
    "recall = true_positive / (true_positive + false_negative)\n",
    "precision, recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC and AUC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TPR and FPR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = []\n",
    "\n",
    "thresholds = np.linspace(0, 1, 101)\n",
    "\n",
    "for t in thresholds: #B\n",
    "    tp = ((y_pred >= t) & (y_val == 1)).sum()\n",
    "    fp = ((y_pred >= t) & (y_val == 0)).sum()\n",
    "    fn = ((y_pred < t) & (y_val == 1)).sum()\n",
    "    tn = ((y_pred < t) & (y_val == 0)).sum()\n",
    "    scores.append((t, tp, fp, fn, tn))\n",
    "\n",
    "df_scores = pd.DataFrame(scores)\n",
    "df_scores.columns = ['threshold', 'tp', 'fp', 'fn', 'tn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>458</td>\n",
       "      <td>726</td>\n",
       "      <td>28</td>\n",
       "      <td>648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>421</td>\n",
       "      <td>512</td>\n",
       "      <td>65</td>\n",
       "      <td>862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>380</td>\n",
       "      <td>350</td>\n",
       "      <td>106</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>337</td>\n",
       "      <td>257</td>\n",
       "      <td>149</td>\n",
       "      <td>1117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>289</td>\n",
       "      <td>172</td>\n",
       "      <td>197</td>\n",
       "      <td>1202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>200</td>\n",
       "      <td>105</td>\n",
       "      <td>286</td>\n",
       "      <td>1269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>99</td>\n",
       "      <td>34</td>\n",
       "      <td>387</td>\n",
       "      <td>1340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>479</td>\n",
       "      <td>1373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn\n",
       "0          0.0  486  1374    0     0\n",
       "10         0.1  458   726   28   648\n",
       "20         0.2  421   512   65   862\n",
       "30         0.3  380   350  106  1024\n",
       "40         0.4  337   257  149  1117\n",
       "50         0.5  289   172  197  1202\n",
       "60         0.6  200   105  286  1269\n",
       "70         0.7   99    34  387  1340\n",
       "80         0.8    7     1  479  1373\n",
       "90         0.9    0     0  486  1374\n",
       "100        1.0    0     0  486  1374"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores['tpr'] = df_scores.tp / (df_scores.tp + df_scores.fn)\n",
    "df_scores['fpr'] = df_scores.fp / (df_scores.fp + df_scores.tn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>458</td>\n",
       "      <td>726</td>\n",
       "      <td>28</td>\n",
       "      <td>648</td>\n",
       "      <td>0.942387</td>\n",
       "      <td>0.528384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>421</td>\n",
       "      <td>512</td>\n",
       "      <td>65</td>\n",
       "      <td>862</td>\n",
       "      <td>0.866255</td>\n",
       "      <td>0.372635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>380</td>\n",
       "      <td>350</td>\n",
       "      <td>106</td>\n",
       "      <td>1024</td>\n",
       "      <td>0.781893</td>\n",
       "      <td>0.254731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>337</td>\n",
       "      <td>257</td>\n",
       "      <td>149</td>\n",
       "      <td>1117</td>\n",
       "      <td>0.693416</td>\n",
       "      <td>0.187045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>289</td>\n",
       "      <td>172</td>\n",
       "      <td>197</td>\n",
       "      <td>1202</td>\n",
       "      <td>0.594650</td>\n",
       "      <td>0.125182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>200</td>\n",
       "      <td>105</td>\n",
       "      <td>286</td>\n",
       "      <td>1269</td>\n",
       "      <td>0.411523</td>\n",
       "      <td>0.076419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>99</td>\n",
       "      <td>34</td>\n",
       "      <td>387</td>\n",
       "      <td>1340</td>\n",
       "      <td>0.203704</td>\n",
       "      <td>0.024745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>479</td>\n",
       "      <td>1373</td>\n",
       "      <td>0.014403</td>\n",
       "      <td>0.000728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn       tpr       fpr\n",
       "0          0.0  486  1374    0     0  1.000000  1.000000\n",
       "10         0.1  458   726   28   648  0.942387  0.528384\n",
       "20         0.2  421   512   65   862  0.866255  0.372635\n",
       "30         0.3  380   350  106  1024  0.781893  0.254731\n",
       "40         0.4  337   257  149  1117  0.693416  0.187045\n",
       "50         0.5  289   172  197  1202  0.594650  0.125182\n",
       "60         0.6  200   105  286  1269  0.411523  0.076419\n",
       "70         0.7   99    34  387  1340  0.203704  0.024745\n",
       "80         0.8    7     1  479  1373  0.014403  0.000728\n",
       "90         0.9    0     0  486  1374  0.000000  0.000000\n",
       "100        1.0    0     0  486  1374  0.000000  0.000000"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_scores[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA9CElEQVR4nO3deXxMd/v/8deVWBKJJYnWHmtQRWKv2osvFbHUUim3qqqtttKi1FJapXX/ai2lLdGqpaWoWquopaglWvuuqH2pnZDP74+Z5E4JQiZzMjPX8/HIo5M5Z87nfdK4cubMOZ9LjDEopZRyfV5WB1BKKeUYWtCVUspNaEFXSik3oQVdKaXchBZ0pZRyE1rQlVLKTWhBV+oJicgREalldQ6l4mhBV5YSkasJvmJF5EaC71uKyGARibF/f0lE1otIRftr24jIXfuyyyKyXUTqW71PACIyVURu37N/L9uXHUmwn6dFZIqI+NuXrRKRm/Zl50RkrojksHZvlKvQgq4sZYzxj/sC/gIiEjw33b7aLPvyp4C1wFwREfuy3+zLsgCfATNFJItz9+KBPk64f8aYWQmWRdhzlwbKAe8lWNbFvqwQ4A+MdF5k5cq0oCuXYYyJAaKA7EDQPctiga8BPyAksdeLSICILBSRsyJy0f44d4Llq0RkqIisE5ErIrJMRLImWP4fETkqIudFpL+D9ukEsBgonsiyS8A8IMwRYyn3pwVduQwRSQ+0AY4bY87ds8wbeA2IAY4+YBNewBQgLxAM3ADG3bPOK/btPA2kA962b78YMAH4D5AT2x+U3CSTiOQB6gHbElkWBLwEHEjuOMozaEFXrqC5iFwCjgFlgEYJlj1nX3YT26mJVsaYM4ltxBhz3hgzxxhz3RhzBfgQqHbPalOMMfuMMTeA2fzv6LgpsNAY86sx5hYwAIh9RO637ef9L4nIuXuWzbPnXgusBoYlWDZGRP4BzgFZga6PGEcpQAu6cg2zjTFZjDFPG2NeMMZsSbBsgzEmCxAALACqPGgjIpJBRD63nza5DPwKZLEf3cc5leDxdWznsMF2VH4sboEx5hpw/hG5R9pzZzHGZL1nWSP783mNMZ3tf0DidDPGZAZK2vcr2e8ElGfQgq7cgjHmKtAZ+I+IlHrAar2AIkAFY0wmoKr9eXnA+gmdBPLEfSMiGbjnPL6jGWP+BD4Axif4EFipB9KCrtyGMeY88AUw8AGrZMR23vySiAQCgx5j898D9UWksoikA4bgnH8/UdjO5zdwwljKxWlBV+5mFFBPREo+YJkvtnPTG4AlSd2oMWYn8CbwLbaj9YvA8WRmTcq4t4Ex2M7ZK/VQog0ulFLKPegRulJKuQkt6Eop5Sa0oCullJvQgq6UUm4ijVUDZ82a1eTLl8+q4ZVSyiVt2bLlnDHmqcSWWVbQ8+XLx+bNm60aXimlXJKIPGiuIj3lopRS7kILulJKuQkt6Eop5SYsO4eulFKOEBMTw/Hjx7l586bVURzKx8eH3LlzkzZt2iS/Rgu6UsqlHT9+nIwZM5IvXz7cZVJKYwznz5/n+PHj5M+fP8mve+QpFxH5SkTOiMiOBywXERkjIgdE5A8RKf0YuZVSKllu3rxJUFCQ2xRzABEhKCjosd91JOUc+lSg7kOWv4ith2MI0B5bmy6llHIadyrmcZ5knx5Z0I0xvwIXHrJKQ2CasdmArQNMjsdOkkTHjx+nb9++HDigbRaVUiohR1zlkosErbmwzRGdK7EVRaS9iGwWkc1nz559osGioqIYMWIEISEh1K5dm1mzZnH58uUn2pZSSiXX+fPnCQsLIywsjOzZs5MrV67470WEsLAwihcvTrNmzbh+/ToA3t7e8c9HRERw6dIlh2RxREFP7H1BopOsG2MmGWPKGmPKPvVUoneuPtKLL74IQGRkJPv27aNFixYEBgZSqVIl3n//fT1yV0o5VVBQENHR0URHR9OxY0feeuut+O/9/PyIjo5mx44dpEuXjokTJwLg6+sb/3xgYCDjx493SBZHFPTjJOi1iK2h7d8O2G6i8uSxDVWhQgUOHTrEqlWr6Nu3L3fv3mXIkCGEhIRQq1Ytpk+fzsqVK1m5ciVr1qzh1q1bKRVJKaUeqUqVKokecFasWJETJ044ZAxHXLa4AOgiIjOBCsA/xpiTDthuorJmzYqPjw/Hjh3D29ubatWqUa1aNT744AP+/vtvvvrqKyZPnkyrVq3+9bqgoCDatGnDG2+8QZEiRVIqnlLKQj169CA6Otqh2wwLC2PUqFHJ2sadO3dYvHgxdev++/qSu3fvsmLFCl5//fVkbT9OUi5bnAH8BhQRkeMi8rqIdBSRjvZVFgGHgAPAZGyd11OMiBAcHMyxY8fuW5YzZ07ee+89Dh06xMaNG1m1ahWrVq1i3rx51KhRg9GjR1O0aFGCgoIoX748kZGRrFmzJiXjKqU82I0bNwgLC6Ns2bIEBwfHF+6454OCgrhw4QK1a9d2yHiPPEI3xkQ+YrnB1jzXaapXr07GjBkfuNzb25vy5cv/67mGDRty6tQpZs+eze7duzl06BC//PILs2bNomvXrgwbNgw/P7+Ujq6USkHJPZJ2tLhz5Q96/p9//qF+/fqMHz+ebt26JXs8l7xT9PPPP3+i12XPnv1fP7SrV6/y7rvvMmbMGH766Sdq1aoFgJeXF6+99hrlypVzSF6llEpM5syZGTNmDA0bNqRTp06PdZt/Yjx6ci5/f3/Gjh3LypUr8ff3Z968ecybN49p06ZRrVo1li5danVEpZSbK1WqFKGhocycOTPZ2xLbGRPnK1u2rHnSBhc//vgjb775JmvXriU4ONjByeD06dPUqVOHXbt28e2339K0aVOHj6GUcozdu3fzzDPPWB0jRSS2byKyxRhTNrH1XfIIPX369Bw7doy//vorRbafLVs2Vq1aRbly5Xj55ZcZMmQI165dS5GxlFLKUVyyoMddi55SBR0gS5YsLFu2jCZNmjBo0CAKFSrEhAkTiImJSbExlVIqObSgP4Sfnx+zZ89m7dq1FCpUiM6dO1OsWDFmzZpFbGxsio6tlFKPyyULur+/PwEBAYlei54SKlWqxK+//srChQvx9fWlRYsWlCtXjq+//ppTp045JYNSSj2KSxZ0gFdeeYVnn33WaeOJCOHh4Wzbto1p06Zx4cIFWrduTY4cOShRogQDBgzg6NEHNuNWSqkU55JXuaQGsbGxREdHs3z5cpYtW8bKlSsBqFu3Lu3bt6d+/fqkSeOSl/kr5VL0Kpf/cdkjdLDNg2AVLy8vSpcuTZ8+fVixYgVHjhxhwIABbN++ncaNG5M3b14GDBjAtm3buHLlimU5lVIpL2463LivI0eOsGrVKjJnzkypUqV45plneP/99wH+9XzRokV5++23HRfEGGPJV5kyZUxyjB071nh7e5urV68mazuOFhMTY+bNm2defPFFIyIG21TCJmvWrKZGjRrmww8/NBs3bjR37tyxOqpSbmHXrl1WRzB+fn73Pbdy5UoTHh5ujDHm6tWrplChQmbz5s3/ev769eumSJEiZu3atYluN7F9AzabB9TVJB2hi0hdEdlr7xvaN5HlASLyg72n6CYRKe64PzmJCwgI4O7du077YDSp0qRJQ8OGDVm0aBFHjhxh9uzZDB8+nMaNG3Px4kX69+9PhQoVyJ8/P1OmTLH0XYZSyjn8/PwoU6YMBw8e/Nfzvr6+hIWFOW/6XBHxBsYDtbHNff67iCwwxuxKsFo/INoY01hEitrXr+mQhA+Q8NLFokWLpuRQTyw4OPi+O1nPnj3Lzz//zKhRo2jbti0jR45k5MiR8Y07lFLJU7169fuea968OZ07d+b69evUq1fvvuVt2rShTZs2nDt37r47w1etWvXIMeNmTwTInz8/P/zww7+Wnz9/ng0bNjBgwAASdmu7ePEi+/fvp2rVqo/esSRIyhF6eeCAMeaQMeY2MBNbH9GEigErAIwxe4B8IpLNIQkfIK5QprYj9Ed56qmniIyMZMOGDXz//ffcuXOHevXqOaxjiVLK+eJmT4yOjv5XMV+zZg2lSpXi//7v/+jbt2/8lXlr1qyhZMmSZM+enfr165M9e3aH5EjKZRiJ9QytcM8624GXgLUiUh7Ii61z0emEK4lIe6A9kOw5WHLlyoWIuFxBjyMiNGnShPDwcF5++WW6dOnCP//8w7vvvuuWHcyVcpaHHVFnyJDhocuzZs2apCPypKpSpQoLFy584PP79u2jcuXKNG7cOP4IPzmScoSelJ6hw4EAEYkGugLbgDv3vcgBPUXjpE2bll69elG2bKJX77gMHx8fvv/+e1q1akX//v3p2bMnt2/ftjqWUsoJChcuzLvvvsuIESMcsr2kHKE/smeoMeYy8BqA2A4vD9u/UtQnn3yS0kM4Rdq0aYmKiiIwMJBRo0axcuVKoqKiCA0NtTqaUiqFdezYkZEjR3L48GHy58+frG0l5Qj9dyBERPKLSDqgBbY+ovFEJIt9GUA74Fd7kU9Rd+/e5fTp049e0QV4eXkxevRofvjhB06dOkXZsmUZOnSoXgWjlAu4evXqfc9Vr1490dMt9z7v6+vLiRMnkl3MIQkF3RhzB+gCLAV2A7ONMTvv6Sv6DLBTRPYALwLdk50sCfr06UP+/PkxFt3tmhIaNWrEzp07adasGQMHDiQiIoJLly5ZHUsp5QKSdG+6MWYRtmbQCZ+bmODxb0CIY6M9Wu7cublx4wYXLlwgKCjI2cOnmKCgIKZPn07VqlXp2rUr5cuXZ968eRQrVszqaEqpVMylb/2Pu1LmyJEj1gZJASJCx44dWblyJZcvX+a5555z6KfvSrkTd3qXHudJ9smlC3rcpDU7d+60OEnKqVy5Mps3byZPnjzUrVuXH3/80epISqUqPj4+nD9/3q2KujGG8+fP4+Pj81ivc+npAENCQvDx8WH79u1WR0lRuXPnZvXq1bz44os0btyYiRMn0rhxYwIDA/WadeXxcufOzfHjx/91B6Y78PHxIXfu3I/1GpefPnfChAkUL16cKlWqOCBV6nb58mUaNGjA6tWrAciUKRO5c+eOn6bXx8eHSpUqUatWLapVq4afn5+VcZVSKeBh0+e6fEH3NLdu3WLZsmUcOHCAQ4cOceLEifi3mpcuXeK3337j1q1bpE2blooVK1K7dm3q1KlDuXLlLE6ulHIEty7o169fZ8uWLYSGhpIpUyYHJHNtN27cYN26dfz8888sX76cbdu2YYzhgw8+oH///lbHU0olk9s2uADYtGkTVatW5bfffrM6Sqrg6+tLrVq1GD58OFu2bOHMmTO0bNmS9957TycAU8rNuXxBj7s93t0/GH1SWbNmZcqUKTRo0IAuXbrwzTffWB1JKZVCXL6gBwQEkCdPHi3oD5E2bVpmzZpFjRo1aNOmDQsWLHj0i5RSLsflCzpAyZIltaA/go+PD/Pnz6d06dI0b948vqm1Usp9uEVBDw0NZc+ePdy6dcvqKKlaxowZWbx4MQULFqRBgwZs2rTJ6khKKQdyVE/RzCLyo4hsF5GdIvKa46M+WNu2bVmzZg3e3t7OHNYlBQUFsXz5cp566ilefPFFfWejlBt5ZEFP0FP0RWyt5iJF5N5Zot4EdhljQoHqwH8TTKeb4goWLEjFihXjb7BRD5czZ05+/vlnMmTIQPXq1Vm/fr3VkZRSDuConqIGyGhvbuEPXCCRjkUpad68eSxatOjRKyoAChQowNq1a8maNSu1a9dm+fLlVkdSSiVTUgp6Yj1Fc92zzjhsc6L/DfwJdDfGxN67IRFpLyKbRWSzo+ddGDZsGP/9738duk13lzdvXtasWUOhQoUIDw+nV69e7N271+pYSqkn5KieonWAaCAnEAaME5H7btt0ZE/Re8Vd6eJOM645Q/bs2Vm1ahVNmjRhzJgxFC1alBo1ajBz5kz9kFkpF5OUgv7InqLY+onONTYHsPUTLeqYiEkTGhrK+fPnOXnypDOHdQsBAQHMmDGDY8eO8dFHH3H06FEiIyPJlSsXb7/9tv5MlXIRDukpCvwF1AQQkWxAEeCQI4M+SunSpQFYu3atM4d1K9mzZ6dv374cOHCAZcuWUb16dUaPHk2xYsX4+uuv9d2PUqmco3qKDgWeF5E/gRVAH2PMuZQKnZgKFSrw9NNPs2bNGmcO65a8vLyoXbs233//PTt37uTZZ5+ldevWNGzY0G2acivljlx+tsWE/v77b3LkyKFNHxzs7t27jBkzhn79+pErVy6WL1/ukA7lSqnH59azLSaUM2dOLeYpwNvbm7feeotVq1Zx4cIFKleuzK5du6yOpZS6h1sVdIA+ffrQrl07q2O4pQoVKrB69WpiY2OpWrUqUVFRnDp1yupYSik7tyvo169fZ/r06Vy9etXqKG6pRIkSrF27lsDAQNq0aUOOHDkoWbIk06dPtzqaUh7P7Qp68+bNuXnzJgsXLrQ6itsqWLAge/bsYcuWLYwYMYJ06dLRqlUrPv30U6ujKeXR3K6gV6pUiRw5cjB79myro7g1Ly8vSpcuTe/evVm3bh1NmzalZ8+eDBo0SC9vVMoiblfQvby8aNasGYsXL+bKlStWx/EI6dOnZ+bMmbz++usMGTKEsmXL0rJlSwYMGMDGjRutjqeUx3C7gg4QGRlJ27ZtiY29bzoZlUK8vb2ZPHkyH3/8MQEBAfz2228MGzaMihUr8tZbb3H9+nWrIyrl9tzqOnSVuly5coV3332X8ePHExISwqRJk6hevbrVsZRyaR5zHfq91qxZw9y5c62O4bEyZszIuHHj+OWXX7hz5w41atQgPDycP/74w+poSrklty7oQ4YMoVOnTnoJo8Vq1KjBzp07GTFiBOvXrycsLIxmzZrxyy+/6AeoSjmQo1rQvSMi0favHSJyV0QCHR/38QwdOpQzZ84wbtw4q6N4PF9fX3r37s2hQ4fo06cPK1asoGbNmhQpUoTJkydrYVfKAR55Dt3egm4fUBvbVLq/A5HGmETv/RaRCOAtY8wLD9uus86hR0REsG7dOg4fPkzmzJlTfDyVNDdu3GDOnDmMHz+eDRs20KpVKyZNmoSvr6/V0ZRK1ZJ7Dj0pLegSigRmPH7MlDFkyBAuXrzIpEmTrI6iEvD19aVVq1asW7eOoUOHMn36dCpXrsxff/1ldTSlXJajWtABICIZgLrAnORHc4xSpUrx8ssvkzFjRqujqER4eXnx3nvvsWDBAg4cOECRIkV49dVXWbdunZ6GUeoxpUnCOklpQRcnAlhnjLmQ6IZE2gPtAYKDg5MU0BFmzpzptLHUk6lfvz5bt25l5MiRTJ8+nWnTphESEkKdOnWoVasWNWrUIFOm+7oaKqUScFQLujgteMjplpTsKfooMTEx/P77704dUz2eggULMmHCBP7++28mT55MoUKF+Oqrr2jUqBE5c+Zk0KBBXL582eqYSqVajmpBh4hkBqoB8x0b0TEGDhxI5cqVOXfOqY2U1BPw9/enXbt2LFq0iAsXLrBy5UpefPFFhgwZQsGCBRk7dix37961OqZSqY6jWtABNAaWGWOupUzU5GnZsiW3b98mKirK6ijqMaRPn57q1avz3XffsWnTJkqWLEm3bt2oWrUq+/btszqeUqmKR936X6lSJc6dO8eePXu0s5GLMsYwffp0unXrxo0bN/jwww/p3r073t7eVkdTyik89tb/e3Xo0IF9+/axcuVKq6OoJyQitGrVip07d1K7dm169epFtWrV9GhdKTysoDdr1owcOXIwY0aquUxePaEcOXIwf/58pk2bxs6dOwkNDeWTTz7RaR6UR/OoUy4ABw8eJH/+/Hh5edTfMrd28uRJOnTowI8//kjGjBlp2bIlnTp1omTJklZHU8rh9JRLAgULFsTLy4vz589z69Ytq+MoB4g7Wl+3bh2NGzdm6tSphIWF6Tsx5XE8rqADHD9+nIIFC+qkXW5ERHj++eeJiorixIkTVK1aldatW2tvWeVRPLKg586dm4oVK/LBBx9w/vx5q+MoBwsMDGTBggXx0/SuXr3a6khKOYVHFnSATz75hMuXLzN8+HCro6gUkClTJhYvXkz+/PmJiIjQphrKI3hsQS9evDiRkZGMHz+eM2fOWB1HpYCsWbOyfPlyMmbMSEREhP5/Vm7PYws62KYDuH37NgsW3DeTgXITuXLlYv78+Zw9e5aXXnpJPwhXbs2jC3rhwoU5cOAA7dq1szqKSkFly5Zl6tSprFu3jg4dOnDnzh2rIymVIjy6oAPky5cPgH/++cfaICpFNW/enIEDBxIVFUXevHkZOHAgR48etTqWUg7lkJ6i9nWq23uK7hQRl7qsYMqUKeTJk4eTJ09aHUWloMGDBzN//nxCQ0P54IMPKFCgAH369OHmzZtWR1PKIR5Z0O09RccDLwLFgEgRKXbPOlmAz4AGxphngWaOj5pyqlatyvXr1xk6dKjVUVQKEhEaNGjAokWLOHz4MG3btuXjjz+mdOnSbNq0yep4SiWbo3qKvgLMNcb8BWCMcanLCQoWLEjHjh2ZNGkSe/bssTqOcoK8efMyefJklixZwpUrV6hYsSJvvPEGx48ftzqaUk/MUT1FCwMBIrJKRLaISOvENiQi7UVks4hsPnv27JMlTiGDBg3Cz8+P3r17Wx1FOVGdOnXYsWMHXbp0ISoqipCQEHr37s2NGzesjqbUY0tKQU9KT9E0QBkgHKgDDBCRwve9yMIWdI/y1FNP0a9fP5YuXcrhw4etjqOcKHPmzIwePZp9+/bRrFkzPvnkE/r2TfSjIqVSNUf1FD0OLDHGXDPGnAN+BUIdE9F5unfvzp49e8ifP7/VUZQF8uXLx7Rp0+jcuTPjxo3DitlAlUoOR/UUnQ9UEZE0IpIBqICtXZ1L8fHxiS/mesWL5xo2bBhPP/20XrOuXI5DeooaY3YDS4A/gE3AF8aYHSkXO2X169ePUqVKcenSJaujKAvEnYLZunUr48ePtzqOUknmcQ0ukmLLli2UK1eOLl26MGbMGKvjKAsYYwgPD2fNmjVs3bqVkJAQqyMpBWiDi8dWpkwZOnXqxPjx49m2bZvVcZQFRITx48eTJk0aSpUqxYQJE4iNjbU6llIPpQX9AT744AOCgoLo3Lmz/kP2UPnz52f79u08//zzdO7cmVq1avH33/deD6BU6qEF/QECAgL4+OOPOXjwIIcOHbI6jrJIcHAwS5cuZdKkSWzatIn69etz7do1q2MplSg9h/4QsbGxXLlyhcyZM1sdRaUCP/30ExEREbz00kvMnj1bG40rS+g59Cfk5eVF5syZuXPnjl6TrAgPD+eTTz5hzpw5DB482Oo4St1HC3oS9OnTh6pVq3LixAmroyiL9ezZk9dee42hQ4cyZ84cq+Mo9S9a0JOgS5cu3L17lwEDBlgdRVlMRJgwYQLlypWjQ4cOnDp1yupISsXTgp4E+fPnp2vXrkydOpXt27dbHUdZLH369EybNo1r167xxhtvYNXnUErdSwt6EvXv35/AwEDq16/Pn3/+aXUcZbGiRYsybNgwFi5cSFRUlNVxlAK0oCdZQEAAP//8MwUKFCBbtmxWx1GpQPfu3alatSrdu3fnr7/+sjqOUlrQH0dYWBirV6/m6aefJiYmhnXr1lkdSVnIy8uLKVOmcPfuXbp162Z1HKUc01PU3k/0H3tP0WgRGej4qKnLwIEDeeGFFzhw4IDVUZSFChQoQP/+/Zk/fz4rVqywOo7ycI+8scjeU3QfUBvbvOe/A5HGmF0J1qkOvG2MqZ/UgV3hxqKHOXnyJIULF6Z69er8+OOPVsdRFrp58ybFihXDz8+Pbdu2kSZNGqsjKTeW3BuLktJT1OPkyJGDgQMHsnDhQhYtWmR1HGUhHx8fRo4cyY4dO5g0aZLVcZQHS8oRelOgrjGmnf37/wAVjDFdEqxTHZiD7Qj+b2xH6zsT2VZ7oD1AcHBwmaNHjzpmLyxy+/ZtSpQogTGGP//8k/Tp01sdSVnEGMMLL7zAn3/+yf79+wkICLA6knJTyT1CT0pP0a1AXmNMKDAWmJfYhlJzT9EnkS5dOkaPHo2vr692OPJwIsKoUaO4ePEi77zzjtVxlIdySE9RY8xlY8xV++NFQFoRyeqwlKlY3bp12bp1K/ny5ePWrVt6k4kHCw0NpXfv3nz55ZfMmDHD6jjKAzmkp6iIZBcRsT8ub9/ueUeHTa28vb25e/cuzZs3p127dsTExFgdSVlk6NChVKpUifbt27Nv3z6r4ygP45CeokBTYIeIbAfGAC2Mhx2qenl5ERYWxldffUWtWrXYu3ev1ZGUBdKkScPMmTNJnz49zZs35+bNm1ZHUh5E50N3sKioKLp3786NGzfo3bs3/fr1w9fX1+pYyskWLVpEeHg4rVq1IioqSudOVw6j86E70auvvsrevXtp3rw5X375Jbdv37Y6krJAvXr1+OCDD/jmm2/o0aOHfrainEILegrIli0bX3/9NTt27CBz5szExMTw9ttv63zqHqZfv3707NmTsWPHakMM5RR6S1sKCgwMBGDLli2MHz+eOXPm8Oeff+Lv729xMuUMIsLIkSO5dOkSQ4YMIW3atPTv3x/79QNKOZweoTvBc889x7Jlyzhy5Ig2yfAwIsKkSZNo2bIlAwYMIDIyUptMqxSjBd1JqlSpQufOnRk9ejQbN260Oo5yIm9vb77++muGDx/O7Nmzef755zl8+LDVsZQb0qtcnOjy5csUK1aMQoUKsWrVKqvjKAssWbKEyMhIsmTJwqZNm3CHO6aVc+lVLqlEpkyZmDt3LrNmzbI6irJI3bp1WbZsGadOneKll17Sq6CUQ2lBd7Ly5cuTLVs2bt++zeDBg7ly5YrVkZSTlStXjilTprB27Vo6deqklzQqh9GCbpHVq1czdOhQwsLC2LBhg9VxlJO1aNGCAQMG8NVXXzFs2DAt6sohtKBbpHbt2qxevZrY2FheeOEFtm3bZnUk5WSDBw8mMjKS9957j9dee02nCVDJpgXdQpUrV2bDhg0EBQXRqFEjzpw5Y3Uk5UReXl588803DB48mKioKKpUqcKxY8esjqVcmEN6iiZYr5yI3LU3xVBJkC1bNubNm0f69Ok5ffq01XGUk3l5eTFo0CDmzZvH3r17iYiI0NMv6ok9sqDbe4qOB14EigGRIlLsAeuNwDYro3oMZcqUYdeuXZQoUYLY2Fj9B+2BGjZsyJgxY9i+fbs2m1ZPzJE9Rbtia0On5w2eQJo0aTDG0LNnT9q0aaNzqnugyMhInn76aUaNGmV1FOWiklLQcwEJT+wdtz8XT0RyAY2BiQ/bkIi0F5HNIrL57Nmzj5vVIwQFBTFt2jQaNGjA1atXrY6jnCh9+vR07tyZn376SefTV0/EUT1FRwF9jDF3H7Yhd+sp6mgiwoABA5g8eTLLli2jZs2aXL582epYyok6depE+vTpGT16tNVRlAtySE9RoCwwU0SOYOte9JmINHJEQE/Url075s6dy9atW2nSpImeU/cgTz/9NC1btiQqKooLFy5YHUe5GIf0FDXG5DfG5DPG5AO+BzobY+Y5OqwnadiwIdOmTaNbt2463aqH6dGjB9evX2fy5MlWR1EuxlE9RVUKiIyMJCIiArDNqa5H6p6hRIkS1KpVi48++ojo6Gir4ygXkqTr0I0xi4wxhY0xBY0xH9qfm2iMue9DUGNMG2PM944O6sm2bNlC+fLlteuNB/nyyy/JlCkTderUYd++fVbHUS5C7xR1AaVLl6ZNmzYMGTKEL774wuo4ygmCg4NZvnw5xhhq166td5CqJNGC7gJEhIkTJ1KnTh06duzIokWLrI6knKBIkSIsXbqUS5cuUbNmTf766y+rI6lUTgu6i0ibNi3fffcdJUuWpFmzZhw9etTqSMoJSpUqxZIlSzhz5gyVK1fW69PVQ2lBdyEZM2Zk8eLFfPTRR+TNm9fqOMpJKlasyKpVq7h16xZVqlTRmTnVA2lBdzHZsmWjW7duAGzbto2ePXty584di1OplBYWFsaaNWvw9fWlZs2a7N+/3+pIKhXSgu7Cli5dyqeffkr9+vX1jlIPULhwYVauXImXlxcRERFcunTJ6kgqldGC7sL69u3L5MmTWbFiBZUqVdLz6h6gQIECzJ07l4MHD/Lyyy/ruzP1L1rQXVy7du1YsmQJx44do1y5chw+fNjqSCqFVa1alQkTJrBs2TK6d+9ObGys1ZFUKqEF3Q3UrFmTDRs2EB4eHv9h6erVq7WlmRtr164dvXr14rPPPqNp06babFwBIFbdTl62bFmzefNmS8Z2dzdv3iRnzpzky5eP7777joIFC1odSaUAYwyjRo3inXfeoUiRIsybN4+QkBCrY6kUJiJbjDFlE1vmkBZ0ItJQRP4QkWj7fOeVkxtaPTkfHx+ioqI4cuQIpUuXZu7cuVZHUilARHjrrbdYtmwZp0+fply5cixevNjqWMpCjmpBtwIINcaEAW0BvT/dYhEREWzdupWiRYvSpEkT2rZty927D52uXrmoF154gc2bN5M/f37Cw8P56KOPdCI3D+WQFnTGmKvmf79BftzfAENZIF++fKxZs4Z3330Xb29vvL29rY6kUki+fPlYt24dLVq0oF+/fkRGRuqHpR7IIS3oAESksYjsAX7CdpSuUoF06dIxbNgwJk2aBEB0dDQdO3bU9nZuKEOGDEyfPp3Bgwcza9Ys5s2bZ3Uk5WSOakGHMeYHY0xRoBEwNNENaU9Ry8Q1yVi7di2TJk2idOnS/P777xanUo4mIvTv35/ChQvz/vvv61G6h3FUC7p4xphfgYIikjWRZdpT1GJdunThl19+4ebNmzz//PN8+OGHem7dzaRJk4YBAwbwxx9/6FG6h3FICzoRKST2Q0ARKQ2kA847OqxyjOrVq7N9+3aaNGnCe++9p63O3FCLFi0oXLgwQ4YM0aN0D+KoFnRNgB0iEo3tipiXjX7MnqoFBAQwY8YM5s2bx+uvvw7A4cOH9R+/m4g7St++fTvz58+3Oo5yEr2xSAFw9epVihYtSnBwMBMmTCA0NNTqSCqZ7ty5w7PPPouPjw9btmwhTZo0VkdSDpDsG4uU+/Pz82PYsGHs37+fMmXK8Omnn1odSSVTmjRpGDp0KH/88QdvvPGGvvvyAFrQFWC7OqJ169bs3buXhg0b0rNnT3r16qVFwMU1b96cwYMHM3XqVHr27Kk3HLk5fQ+m/iUwMJDZs2fTo0cP9u/fT2xsLF5e+nfflQ0cOJCLFy8yevRoAgICGDRokNWRVArRc+gqUcYYYmJiSJcuHRs3bmTx4sV06NCBHDlyWB1NPYHY2Fhef/11pk6dyvr166lYsaLVkdQT0nPo6rGJCOnSpQNg5cqVvP/+++TNm5d27dpp93kX5OXlxbhx4wgMDGT48OFWx1EpRAu6eqS+ffuyf/9+OnTowNdff01ISAj9+/e3OpZ6TH5+fnTt2pUFCxawa9cuq+OoFKAFXSVJoUKFGDt2LAcOHODVV18lQ4YMAMTExPDf//6XY8eOPWILKjXo0qULGTJk4OOPP7Y6ikoBeg5dJcu6deuoXLky3t7eNGrUiK5du1K1atX4uWNU6tO9e3c+++wzDh48SHBwsNVx1GPSc+gqxVSqVIlDhw7Rq1cvVq5cSfXq1QkNDeXEiRNWR1MP0LNnTwC918ANaUFXyZY/f35GjBjBsWPH+OKLLyhYsGD81TDr16/XqXpTmbx58/LKK68wadIk/cPrZvSUi0ox165dI0+ePPj4+PDhhx/y6quv6jXtqcT+/fspXbo0RYsWZfXq1fGfiajUzxk9RVvae4r+ISLrRUQnAlH4+fmxaNEi8ubNS9u2bQkJCaFv37562WMqEBISwrfffsuWLVto27at3kHqJhzVU/QwUM0YUxJbc4tJjg6qXNNzzz3H+vXrmTVrFoUKFWLkyJGcP2+bWXn37t38/vvvWkwsEhERwfDhw5k1axZDhybak0a5GmPMQ7+AisDSBN+/C7z7kPUDgBOP2m6ZMmWM8jznzp0zsbGxxhhjXn/9dQOYYsWKme+++87cvXvX4nSeJzY21rRu3doAZuPGjVbHUUkAbDYPqKsO6ymawOvA4sQWaAs6FRQUFH9J48cff8yXX36JMYZmzZpRtmxZli1bZnFCzyIijB8/nowZMzJ27Fir46hkclhPUQARqYGtoPdJbLnRFnQqgcDAQNq2bcuff/5JVFQUly5d4pdffrE6lsfx9/fn1VdfZfbs2eiBlmtzWE9RESkJfAE0NMZo+zmVZN7e3rRu3Zo9e/YwYMAAABYvXkzVqlWZOHEip0+ftjih++vcuTO3b9/myy+/tDqKSgZH9RQNBuYC/zHG7HN8TOUJ0qVLh5+fHwD//PMPZ86coVOnTuTMmZOaNWsybdo07ty5Y3FK9/TMM89Qo0YNJk6cqE3DXZijeooOBIKAz0QkWkT0AnOVLC1atGD37t388ccf9O/fn6NHj/L+++/j7e0NwM6dO4mJibE4pXt58803OXr0KIsWLbI6inpCemORcgnGGE6cOEHu3LmJiYkhW7ZsADRt2pQ+ffpQsGBBixO6vpiYGPLly0eJEiVYsmSJ1XHUA+hcLsrliQi5c+eOfzx16lTCw8P5+uuvKVq0KB06dNDb2JMpbdq0dOjQgaVLl7J3716r46gnoAVduZw0adLQoEEDvv76aw4ePEj79u2ZMmWKTuHrAB06dMDf358+fRK9UE2lclrQlUvLmTMn48eP59ixYzz33HMA9OrViwEDBnD9+nWL07mebNmy0a9fP+bPn8/PP/9sdRz1mLSgK7cQd07dGMPp06f54IMPCA0NZc2aNRYncz1vvfUW+fPnp0ePHnpVkYvRgq7ciojwzTffsGLFCu7cuUO1atV488034+ePUY/m4+PDyJEj2blzJ5Mm6bRMrkSvclFu6+rVq7z77rtMnDiRU6dOERQUxJIlSzhw4ABhYWGEhoaSMWNGq2OmSsYYXnjhBf744w/27dtHUFCQ1ZGUnV7lojySv78/Y8eO5fTp0/EF6csvv6Rr165UqVKFLFmyEB4ezpw5cyxOmvqICKNHj+bq1atERERw7do1qyOpJNCCrtxeYGBg/OPZs2dz/PhxfvzxR9555x22b9/OuHHj4pdfuHDBioipUsmSJZkxYwYbN26kcePG3Lp1y+pI6hH0lIvyaHfv3uXs2bNkz56dkydPUqBAAerUqcMbb7xBnTp1SJMmjdURLTdlyhTatm1LkyZNmDlzpv5MLKanXJR6AG9vb7Jnzw7Ybqzp1asX69evp379+gQHB9OnTx9OnjxpcUprvfbaa4waNYo5c+ZoI4xUTo/QlbrH7du3WbRoEVOmTGHZsmXs3r2bfPnyERMTQ9q0aa2OZ5nWrVvz7bffsnHjRsqUKWN1HI/1sCP0JBV0EakLjAa8gS+MMcPvWV4UmAKUBvobY0Y+apta0JUr+Oeff8icOTMA9erVY8eOHRQqVIhChQpRo0YN6tev7zFXyly8eJESJUqQOXNmtmzZgo+Pj9WRPFKyTrkksafoBaAb8MhCrpQriSvmsbGxvPDCC1SvXp2bN2/y/fff88orr9C0aVPAdpnfRx99xLhx49izZ4+VkVNMQEAAX375Jbt27WLgwIFWx1GJSMqnG+WBA8aYQwAiMhNoCOyKW8EYcwY4IyLhKZJSKYt5eXnx9ttvx38fGxvLunXr4tvp3bhxg379+sWv26pVKwYNGkSBAgUsyZtS6tSpQ4cOHRg5ciQlS5akZcuW8T8DZb2U6Cn6QNpTVLkLLy8vqlSpQuXKlQHw9fXlxo0bHD16lJ49ezJ79myKFCnCtm3bLE7qeCNHjuS5557jP//5D02aNNGOUqmIQ3uKPor2FFXuSkTw8fEhODiYTz75hIMHD9KjRw+effZZAD766CMiIiIYMWIE69ev5/bt2xYnfnL+/v6sWbOGESNGsGjRIooVK8batWutjqVwYE9RpdT/5MyZk08++YR06dIBtnPs+/fvp2/fvlSqVInAwEBat25tccon5+3tTe/evdm2bRuBgYE0a9ZMj9RTAYf0FFVKPVy/fv3Ys2cPp0+fZs6cOfznP/8hS5Ys8cv79+/P2rVrseoy4if1zDPPMGfOHC5dukSrVq20H6nVjDGP/ALqAfuAg9guSwToCHS0P86O7Uj+MnDJ/jjTw7ZZpkwZo5Qy5sSJEyZLliwGMOXKlTNLliwxsbGxVsd6LJMmTTKAGTp0qNVR3B6w2TygruqNRUqlAteuXeObb77ho48+4ujRo1SpUoVp06aRL18+q6MliTGGVq1aMXPmTL744gteffVVvLz0RvSUoLf+K5XK+fn50aFDB/bu3cu4ceO4fPkyWbNmBWzTAKd2IsLEiRMpV64cbdu2pXTp0ixevNjlTiG5Oi3oSqUi6dOn580332Tbtm34+/tz584dypYtS0REBPPnzycmJsbqiA+UMWNG1q9fz7fffsuVK1eoV68ePXr0sDqWR9GCrlQqFHezzp07d4iMjGTz5s00atSIXLly8eabbxIdHW1twAfw8vIiMjKS3bt307VrV8aMGcPEiROtjuUxtKArlYr5+PgwaNAgjh07xo8//kiVKlWYOnUqhw8fBuDQoUPxp2hSk3Tp0vHpp58SHh5Oly5d+OWXX6yO5BH0Q1GlXExMTAzGGNKlS8f/+3//j169epExY0aaNm1K8eLFCQkJoV69enh7e1sdlcuXL1OxYkVOnjzJxo0bCQkJsTqSy0v2bIspQQu6Uo6xadMmxo4dy8KFC7l06RL+/v5cvnwZEWHOnDmkT5+eWrVqWTY74qFDhyhfvjyZM2dmzZo15MyZ05Ic7kILulIewBjDhQsXOHHiBCVLlgSgTJkybN26lYwZMxIREUGTJk2oVq2a05s+b9q0iZo1a5I3b15Wr16tTaeTQS9bVMoDiAhBQUHxxRxg/fr1LFmyhObNm7NkyRKaNGlCz549AdsfgJ9++skp59/Lly/PggULOHDgAPXq1ePKlSspPqYn0iN0pTxETEwMGzZswN/fn1KlSrF3716KFi1KmjRpqFChAgUKFCB79uy88sorhIWFce7cOX7//XeqVatGhgwZHJJhwYIFvPTSS5QqVYq5c+eSJ0+eR79I/YseoSulSJs2LVWqVKFUqVIAFChQgNWrV/P2228TGxvLr7/+ypgxYzh48CAA0dHR1KtXjzx58tCnTx+OHDmS7AwNGjTghx9+YO/evZQtW5Zff/012dtU/6NH6EqpeMYYYmNj8fb25vLly/z2229MnjyZefPmcffuXXx9fTl//jy+vr6MHz+en3/+mezZs/PMM88QFhZGaGhofJenh9m9ezeNGjXi0KFDdO7cmbp161K1alX8/PycsJeu7WFH6EnpWJSUnqJiX14PuA60McZsTVZqpZTTiUj85Y6ZMmWiTp061KlTh2PHjjFjxgzOnj0bf7XM1atXOXjwIL/++isXLlwAIHv27Jw8eRKARo0asWfPHipXrkzlypWpUqUKBQoUQER45pln2LRpE506deLzzz9nzJgxpE2bltq1a9O+fXvCw8NJkyZJ5Ukl8MgjdHtP0X1AbWyzKP4ORBpjdiVYpx7QFVtBrwCMNsZUeNh29QhdKfdgjOHUqVNs27aNw4cP8+abbwIwbdo0vvvuO9atW8fFixcBqFmzJj///DMAmzdvpkCBAvj6+rJ27VqWLVvGt99+y99//03OnDmJiIigYMGCFChQIP6/mTJlsmw/U4vkHqE/sqeo/ftp9qkdN4hIFhHJYYw5mczsSqlUTkTIkSMHOXLk+NfzrVu3pnXr1sTGxrJr1y7Wrl0bf3QfGxtLjRo1uHr1KmnTpo1/Tfv27alVqxaff/45n3/++X1j+fn5ERwcTGxsbKLn9AMDAwkICCAmJoa//vrrvuVBQUFkyZKFW7ducfz48fuWP/XUU2TKlImbN29y4sSJ+5Zny5YNf39/rl+/Hv9OJKEcOXKQIUMGrl69mmjDj5w5c+Lr68vrr78ef7WRIyWloCfWU/Teo+8H9R391x6LSHugPUBwcPDjZlVKuSAvLy+KFy9O8eLF458zxvDDDz+wbdu2+KN3gOeee44GDRoQERFBnz59OH/+PKdPn+bixYtcu3aNrFmzEhQURExMTKKzUObJk4fcuXNz69Ytrl+/ft/yfPnykSNHDq5fv87NmzfvW16gQAGefvpprly5wq1bt+5bXrBgQYKCgrh06VKiE6UVKlSILFmycP78+USbfYSEhJApUyayZcv24B9YMiSloCelp2iS+o4aYyYBk8B2yiUJYyul3JC3tze1atWiVq1aD1w+cuRIJ6dyfY7qKap9R5VSymKO6im6AGgtNs8B/+j5c6WUcq5HnnIxxtwRkS7AUmyXLX5ljNkpIh3tyycCi7Bd4XIA22WLr6VcZKWUUolJ0oWexphF2Ip2wucmJnhsgDcdG00ppdTj0Fv/lVLKTWhBV0opN6EFXSml3IQWdKWUchOWzbYoImeBo0/48qzAOQfGcYWxdZ89Y2xPG9fKsV11n/MaY55KbIFlBT05RGTzgyancdexdZ89Y2xPG9fKsd1xn/WUi1JKuQkt6Eop5SZctaBP8sCxdZ89Y2xPG9fKsd1un13yHLpSSqn7ueoRulJKqXtoQVdKKTeRqgu6iNQVkb0ickBE+iayXERkjH35HyJS2knjFhWR30Tkloi87YgxH2PslvZ9/UNE1otIqJPGbWgfM1pENotIZUeMm5SxE6xXTkTuikhTZ4wrItVF5B/7PkeLyEBHjJuUsROMHy0iO0VktTPGFZF3EuzvDvvPO9BJY2cWkR9FZLt9nx0ya2sSxg0QkR/sv9+bRKR4Ytt5gnG/EpEzIrLjAcsdX7+MManyC9tUvQeBAkA6YDtQ7J516gGLsXVMeg7Y6KRxnwbKAR8Cbzt5n58HAuyPX3TiPvvzv89cSgJ7nLXPCdb7Bdusn02dtM/VgYUW/W5nwda3Nzjud85ZP+sE60cAvzhxn/sBI+yPnwIuAOmcMO4nwCD746LACgftc1WgNLDjAcsdXr9S8xF6fHNqY8xtIK45dULxzamNMRuALCKS494NOXpcY8wZY8zvwP1NBVN+7PXGmLgmjBuwdYdyxrhXjf23EPAjkRaDKTW2XVdgDnDGyeOmhKSM/Qow1xjzF9h+55w0bkKRwAwHjJvUsQ2QUUQE2wHEBeCOE8YtBqwAMMbsAfKJSLKbfhpjfsW2Dw/i8PqVmgv6gxpPP+46KTFuSnncsV/H9hfeKeOKSGMR2QP8BLR1wLhJGltEcgGNgYk4TlJ/1hXtpwAWi8izThy7MBAgIqtEZIuItHbSuACISAagLrY/oo6QlLHHAc9ga1/5J9DdGBPrhHG3Ay8BiEh5IC+OOVByRLbHkpoLusOaU6fAuCklyWOLSA1sBb2Ps8Y1xvxgjCkKNAKGOmDcpI49CuhjjLm/jXrKjrsV27wZocBYYJ4Tx04DlAHCgTrAABEp7IRx40QA64wxDzvCdPTYdYBoICcQBowTkUxOGHc4tj+e0djeCW4j+e8MksLhtSZJHYssYlVzaisbXidpbBEpCXwBvGiMOe+sceMYY34VkYIiktUYk9zJjZIydllgpu2dOFmBeiJyxxgzLyXHNcZcTvB4kYh85sR9Pg6cM8ZcA66JyK9AKLAvhceN0wLHnW5J6tivAcPtp/YOiMhhbOe0N6XkuPb/z6+B7YNK4LD9K6U5vtY44uR/Snxh+2NzCMjP/z7MePaedcL594cKm5wxboJ1B+PYD0WTss/B2Hq3Pu/kcQvxvw9FSwMn4r531s/bvv5UHPOhaFL2OXuCfS4P/OWsfcZ26mGFfd0MwA6guDN+1kBmbOd+/Zz8OzYBGGx/nM3+O5bVCeNmwf7hK/AGtvPajtrvfDz4Q1HH1y9HBU+JL2yfAu/D9il1f/tzHYGO9scCjLcv/xMo66Rxs2P763oZuGR/nMlJY38BXMT21jQa2OykcfsAO+1j/gZUdtb/53vWnYoDCnoS97mLfZ+3Y/sA2pF/RB+5z8A72K502QH0cOK4bYCZjtrXx/h55wSW2f8t7wBaOWncisB+YA8wF/tVZA4YdwZwEtvFE8exnSJN0fqlt/4rpZSbSM0fiiqllHoMWtCVUspNaEFXSik3oQVdKaXchBZ0pZRyE1rQlUsQkaAEswCeEpET9seXRGRXCow3WB5zJk0RufqA56c6aoZIpR5GC7pyCcaY88aYMGNMGLY5XT61Pw4DHjnfh4ik5ruilXIILejKHXiLyGT7HNrLRMQXwD6x1TD7XOLdRaSMiKy2T3a1NG5mOxHpJiK77HNSz0yw3WL2bRwSkW5xT4pIT/tc4TtEpMe9YezzXI+zb/MnbNMtxy0bnmCskSn1A1GeSY9alDsIASKNMW+IyGygCfCNfVkWY0w1EUkLrAYaGmPOisjL2Oazbwv0BfIbY26JSJYE2y0K1AAyAntFZAK2ueBfAypgu9Nvo4isNsZsS/C6xkARoAS2W9h3AV/ZG0U0BooaY8w9YymVbFrQlTs4bIyJtj/egm3+jDiz7P8tAhQHltsn+fLGdls2wB/AdBGZx79nVPzJGHMLuCUiZ7AV58rAD8Y2aRYiMheogm2GvjhVgRnGNjvk3yLyi/35y8BN4Av7kfvCJ99lpe6np1yUO7iV4PFd/n2gcs3+XwF2xp2HN8aUMMb8n31ZOLY5NcoAWxKcb09su4lNeZqYxKYfvoNtkq852KYgXpLEbSmVJFrQlafYCzwlIhUBRCStiDwrIl5AHmPMSqA3tpn3/B+ynV+BRiKSQUT8sJ1CWZPIOi1ExNt+nr6GfUx/ILMxZhHQA9sHuko5jJ5yUR7BGHPbfungGBHJjO13fxS2Wfi+sT8n2K6euWQ/LZPYdraKyFT+N0f3F/ecPwf4AXgB2wx6+7Cduwfbufj5IuJjH+stB+2eUgA626JSSrkLPeWilFJuQgu6Ukq5CS3oSinlJrSgK6WUm9CCrpRSbkILulJKuQkt6Eop5Sb+P/ALolaPFfA7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(df_scores.threshold, df_scores.tpr, color='black', linestyle='solid', label='TPR')\n",
    "plt.plot(df_scores.threshold, df_scores.fpr, color='black', linestyle='dashed', label='FPR')\n",
    "plt.legend()\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "plt.yticks(np.linspace(0, 1, 11))\n",
    "\n",
    "plt.xlabel('Thresholds')\n",
    "plt.title('TPR and FPR')\n",
    "\n",
    "# plt.savefig('04_fpr_tpr_plot.svg')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tpr_fpr_dataframe(y_val, y_pred):\n",
    "    scores = []\n",
    "\n",
    "    thresholds = np.linspace(0, 1, 101)\n",
    "\n",
    "    for t in thresholds:\n",
    "        tp = ((y_pred >= t) & (y_val == 1)).sum()\n",
    "        fp = ((y_pred >= t) & (y_val == 0)).sum()\n",
    "        fn = ((y_pred < t) & (y_val == 1)).sum()\n",
    "        tn = ((y_pred < t) & (y_val == 0)).sum()\n",
    "\n",
    "        scores.append((t, tp, fp, fn, tn))\n",
    "\n",
    "    df_scores = pd.DataFrame(scores)\n",
    "    df_scores.columns = ['threshold', 'tp', 'fp', 'fn', 'tn']\n",
    "\n",
    "    df_scores['tpr'] = df_scores.tp / (df_scores.tp + df_scores.fn)\n",
    "    df_scores['fpr'] = df_scores.fp / (df_scores.fp + df_scores.tn)\n",
    "\n",
    "    return df_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>440</td>\n",
       "      <td>1236</td>\n",
       "      <td>46</td>\n",
       "      <td>138</td>\n",
       "      <td>0.905350</td>\n",
       "      <td>0.899563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>392</td>\n",
       "      <td>1101</td>\n",
       "      <td>94</td>\n",
       "      <td>273</td>\n",
       "      <td>0.806584</td>\n",
       "      <td>0.801310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>339</td>\n",
       "      <td>972</td>\n",
       "      <td>147</td>\n",
       "      <td>402</td>\n",
       "      <td>0.697531</td>\n",
       "      <td>0.707424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>288</td>\n",
       "      <td>849</td>\n",
       "      <td>198</td>\n",
       "      <td>525</td>\n",
       "      <td>0.592593</td>\n",
       "      <td>0.617904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>239</td>\n",
       "      <td>723</td>\n",
       "      <td>247</td>\n",
       "      <td>651</td>\n",
       "      <td>0.491770</td>\n",
       "      <td>0.526201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>193</td>\n",
       "      <td>579</td>\n",
       "      <td>293</td>\n",
       "      <td>795</td>\n",
       "      <td>0.397119</td>\n",
       "      <td>0.421397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>152</td>\n",
       "      <td>422</td>\n",
       "      <td>334</td>\n",
       "      <td>952</td>\n",
       "      <td>0.312757</td>\n",
       "      <td>0.307132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>98</td>\n",
       "      <td>302</td>\n",
       "      <td>388</td>\n",
       "      <td>1072</td>\n",
       "      <td>0.201646</td>\n",
       "      <td>0.219796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>57</td>\n",
       "      <td>147</td>\n",
       "      <td>429</td>\n",
       "      <td>1227</td>\n",
       "      <td>0.117284</td>\n",
       "      <td>0.106987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn       tpr       fpr\n",
       "0          0.0  486  1374    0     0  1.000000  1.000000\n",
       "10         0.1  440  1236   46   138  0.905350  0.899563\n",
       "20         0.2  392  1101   94   273  0.806584  0.801310\n",
       "30         0.3  339   972  147   402  0.697531  0.707424\n",
       "40         0.4  288   849  198   525  0.592593  0.617904\n",
       "50         0.5  239   723  247   651  0.491770  0.526201\n",
       "60         0.6  193   579  293   795  0.397119  0.421397\n",
       "70         0.7  152   422  334   952  0.312757  0.307132\n",
       "80         0.8   98   302  388  1072  0.201646  0.219796\n",
       "90         0.9   57   147  429  1227  0.117284  0.106987\n",
       "100        1.0    0     0  486  1374  0.000000  0.000000"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.random.seed(1)\n",
    "y_rand = np.random.uniform(0, 1, size=len(y_val))\n",
    "df_rand = tpr_fpr_dataframe(y_val, y_rand)\n",
    "df_rand[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7v0lEQVR4nO3deXxM5/fA8c9JIrWHpnYqKtaoRAVVaqnat6pS1dL62lJqqVpq36poKVWptZYWUVvxU6F21VJJCEJrSW3pal8aW+L5/TE3OgiiJplkct6v17zMzHPnPudOxpk7z733OWKMQSmlVNrn5uwAlFJKOYYmdKWUchGa0JVSykVoQldKKRehCV0ppVyEJnSllHIRmtBVkonIMRF50ckxZBKR/xORCyKyOJn6GCYi85Jj3Y4mIjVEJMbZcTwqEdksIh2SuKwREd/kjikt0oSewkTkst3tpohcsXv8upVMbliPz4vIjyJS2XrtWyISb7VdFJE9ItLI2dsEICJzROT6Hdv3qtV2zG47/xKR2SKS1WrbLCJXrbbTIrJMRPLdp6tXgDyAtzGmhQPidomEqBRoQk9xxpisCTfgBNDY7rn51mJfW+25gG3AMhERq2271ZYD+BxYKCI5UnYr7ukj++0zxnxt19bYivsZoAIwyK7tHavNF8gKjLtPH4WBQ8aYuIcNTkQ8HvY1jiYi7s6OQbkuTeipmDHmBjAXyAt439F2E/gKyAIUS+z1IpJTRFaJyCkROWfdL2jXvllERorIDyJySUS+E5En7NrbiMhxETkjIgMdtE2/AaFAmUTazgPLgYB7bM9wYAjwqrVH315E3ERkkBXn3yLypYh4Wcv7WD/P24vICWDjHevLYsWS3+5XRX6r2dNa1yUR2S8igXavyy8iS6339aiIdL/X9lq/XKaIyGoR+QeoKSINRWS39SvrpIgMs1s+IeY3ReSE9atloF17Jmud50TkALYvR/v+Sll/1/NW3E3uiOVzEQm1tvUHEckrIhOt9f0iIuXusy1GRLqIyGHrfRkpIkVFZLu1LYtExNNu+Y4ickREzorISrv3FhGpbfV3QUQmA3JHX/8TkZ+tuNaKSOF7xaX+pQk9FRORx4C3gBhjzOk72tyBdsAN4Pg9VuEGzMa2V/skcAWYfMcyra315AY8gd7W+ksDU4A2QH5sXygFeUQiUghoAOxOpM0beBk4kthrjTFDgQ+xfsEYY77A9v68BdQEnsK2h3/nNlYHSgF171jfP0B94He7XxW/W81NgIXYfgmtTFiniLgB/wfsAQoAtYCeInLbuu/QGhgFZMP2i+sfoK217obA2yLy0h2vqQqUsNY/RERKWc8PBYpat7rAmwkvEJEMVmzfYft7dgPmi0gJu/W2xPbr6AngGrAd2GU9XgJ8cp/tAKgHlAeeBfoC04HXgULYvqRfs2J5ARht9ZcP22d0odX2BLDULo5ooIrddrwEDMD2WcgFfA+EPCAuBWCM0ZuTbsAx4MU7nhsGXAfOA39j26ssb7W9BcRZbTewJeiWD9FfAHDO7vFmYJDd4y7AGuv+EGChXVsWK64X77HuOcBVK7bzwOk7tvOy9fxxbENFmexiiAUuAAaIBJ68zzYMA+bZPd4AdLF7XMJ6bzwAH2udT91nfTWwfWHe2cd6u8elgSvW/UrAiTuW7w/Mvs/78uUD/i4TgQnW/YSYC9q17wRaWfd/BerZtXVKiB94HvgTcLNrDwGG2cUyw66tG/Cz3eOngfP3idMAVeweRwD97B6PByZa97/ANgSX0JbV+rv4YPsy22HXJkAM0MF6HAq0t2t3sz4jhe3i8E2J/6Np7aZ76KnTImNMDmNMbmPMC8aYCLu2HcaYHEBObHuOz99rJSKSWUSmWcMRF4GtQI47xnH/tLsfi+0/Htj2yk8mNBjb3uyZB8Q9zoo7hzHmiTvaXrKeL2yM6WKMuWLX1t0Y4wWUtbbrYX4J5Of2XyjHsSXzPHbPneTh3fm+ZLTG4AtjG6I5n3DDtjeZJ5F1JNq/iFQSkU3WkM0FIAjbnur9+k/078Lt254fOGlsw3H27QXsHv9ld/9KIo+zcn9Jff1tfxdjzGVsn58Cd26DsWVp+20qDHxq9/6exZb07bdDJUITehpl/QfpArS5z7jne9j2WCsZY7ID1azn5R7L2/sD289o2wtEMnPHOL6jGWP2AR8AwSKSlBgBfseWABI8ie1XjH2iud+Uog873ehJ4KjdF1cOY0w2Y0yDh+hjAbYv40LWF9lUkvY3gTv+Lti2N8HvQCFrWMi+/bckrtuRbvu7WMcrvK1Y7vxsCbdv00mg8x3vcSZjzI8pE3rapQk9DTPGnAFmYhseSUw2bHtN50XkcWzjr0m1BGgkIlWtA10jSJnPy1xs479NHrSgJQR4V0SKiO1UyIQx9qSeBfMX4J1wIDUJdgIXRaSfdYDSXUTKiEiFB77yX9mAs8aYqyJSEdsYe1ItAvqL7YB3QWzDJgl+wjY+31dEMohIDaAx1th1ClsAtBORAOtY0IfAT8aYY8C3gJ+IvGz96umO7cB/gqnYttEPQES8ROSRT1FNDzShp30TgQYiUvYebZmA08AOYE1SV2qM2Q90xfYf8w/gHLZxzmRljLkOTAIGJ/Els7Cd7bMVOIptHL/bfV9xe3+/YPtS+NX6iZ//AcvHY0uSAVZ/p7F9qSb1CwFsv6xGiMglbF/Gix7itcOxDWUcxXbw8yu72K5j+yKsb8X1OdDW2sYUZYzZgO1vuBTb56co0MpqOw20AMZgG4YpBvxg99pvgLHYTsm9CERh2yb1AGIdZFBKKZXG6R66Ukq5CE3oSinlIjShK6WUi9CErpRSLsJpkxU98cQTxsfHx1ndK6VUmhQREXHaGJMrsTanJXQfHx/Cw8Od1b1SSqVJInKvuZt0yEUppVyFJnSllHIRmtCVUspFOL2Ci1JKPYobN24QExPD1atXnR2KQ2XMmJGCBQuSIUOGJL9GE7pSKk2LiYkhW7Zs+Pj4kPRJOlM3YwxnzpwhJiaGIkWKJPl1DxxyEZFZYivtFXWPdhGRSVapqb0i8sxDxK2UUo/k6tWreHt7u0wyBxARvL29H/pXR1LG0OdgKzt1L/WxzZZWDFv1lCkPFYFSSj0iV0rmCf7LNj0woRtjtmKrGHIvTbGV2DLGmB3YKuLke+hIkujUqVN06NCB48fveSqmUkqlS444y6UAt5ePiuEepaJEpJOIhItI+KlTp/5TZxs2bOCLL77gqaeeol+/fuj0v0opZzpz5gwBAQEEBASQN29eChQocOuxiBAQEECZMmVo0aIFsbGxALi7u996vnHjxpw/f94hsTgioSf2uyDRLGuMmW6MCTTGBObKleiVqw/UqlUr5s6di6enJx999BEFCxbUvXWllNN4e3sTGRlJZGQkQUFBvPvuu7ceZ8mShcjISKKiovD09GTq1KkAZMqU6dbzjz/+OMHBwQ6JxREJPYbb6wEWxFZPMNm0bduWM2fOULt2bX7//XfKlCnDsWPHkrNLpZR6JM8//zxHjhy56/nKlSvz22+OKfvqiNMWVwLviMhCoBJwwRjzhwPWe1+ZM2fmu+++491332XixIkMGDCAefPm4eam10oplV717NmTyMhIh64zICCAiRMnPtI64uLiCA0NpV69288viY+PZ8OGDbRv3/6R1p/ggQldREKAGsATIhKDrdBwBgBjzFRgNdAAOALEAu0cElkSTZgwgZw5czJ06FBy5MhB586d8ff3T8kQlFIqUVeuXCEgIACw7aEnJO6E548dO0b58uWpXbu2Q/p7YEI3xrz2gHaDrZiw0wwePJjz588zYcIEpk6dSuvWrfn444/Jly/ZTrZRSqVCj7on7WgJY+X3ev7ChQs0atSI4OBgunfv/sj9ucT4hIgwfvx4pk+fTrZs2Zg/fz4FChSgWbNmrF27lps3bzo7RKWUuouXlxeTJk1i3Lhx3Lhx45HX5xIJHWxJvWPHjvzxxx/UrVsXY8ytMStfX19Gjx7Nn3/+6ewwlVLqNuXKlcPf35+FCxc+8rrEWedxBwYGmuQqcHHz5k1GjRrFZ599xtChQ1m6dCmbNm3Cw8ODpk2bMnr0aIoVK5YsfSulUtbPP/9MqVKlnB1Gskhs20QkwhgTmNjyLpnQE1y4cAEvLy/i4+P58MMPOXv2LLNnzwYgJCSE+vXrJ2v/Sqnkpwn9Xy4z5JIYLy8vANavX8+QIUNYvnw5o0aNonDhwjRs2JDRo0frlaZKKZfh0gk9Qd26dVm/fj3Zs2fnnXfeIWfOnDRo0IABAwbw3HPPsWXLFmeHqJRSjyxdJHSAWrVqsWvXLqZMmcKePXs4fvw4X3zxBSdPnqRGjRo0aNCAGTNmsGHDBr3qVCmVJqWrAhfu7u4EBQXRrFkz/vjjDwICAmjcuDE9e/YkNDSU0NDQW8v27NmTCRMmODFapZR6OOlmD91enjx5bl29FRISwoIFCyhWrBhr165l06ZNtGvXjokTJ7JkyRLnBqqUUg8hXSZ0e++88w6zZ8/m+PHjNGvWjFOnTjFt2jQqVapE+/btiY6OdnaISqlULmE63ITbsWPH2Lx5M15eXpQrV45SpUoxfPhwgNueL1myJL1793ZYHOk+obu5ufHWW2+xe/duAgICaNmyJdOnT+frr7/Gzc2Nli1bcu3aNWeHqZRKxRIu5U+4+fj4ALb5W3bv3k14eDjz5s0jIiLitud3797NqlWr+OGHHxwSR5ISuojUE5GDVt3Q9xNpzyki31g1RXeKSBmHRJeC8uXLx6ZNm3j33XepX78+hQsX5vPPP2fXrl1UqlSJ6dOnc+nSJWeHqZRKg7JkyUL58uXv+sWfKVMmAgICUm76XBFxB4KB2tjmPg8TkZXGmAN2iw0AIo0xzUSkpLV8LYdEmII8PT355JNPbj2eP38+vr6+nD17ls6dO/Pee+9RpUoVnnrqKZ566imqV69OYGCgS9YzVCqtqlGjxl3PtWzZki5duhAbG0uDBg3uan/rrbd46623OH36NK+88sptbZs3b35gn/azKhYpUoRvvvnmtvYzZ86wY8cOBg8ejH21tnPnznH48GGqVav24A1LgqSc5VIROGKM+RXAmve8KWCf0EsDowGMMb+IiI+I5DHG/OWQKJ3AGEOTJk0YOXIkMTExiAiZMmXi8OHDhIWFcfasrcxqQEAAnTt3plGjRuTPn1/nY1cqHbrXrIrff/895cqVw83Njffffx8/Pz82b97M999/T9myZTl48CDvv/8+efPmdUwgxpj73oBXgJl2j9sAk+9Y5kPgE+t+RSAOKJ/IujoB4UD4k08+adKCuLg489NPP5nhw4cbPz8/M2HCBGOMMadPnzZTpkwxAQEBBlvJPfPYY4+ZkiVLmiFDhpj4+HjnBq5UOnHgwAFnh2CyZMly13ObNm0yDRs2vO/zBw8eNLly5TK7d+9OdL2JbRsQbu6Rr5Oyh56UmqFjgE9FJBLYB+y2kvqdXx7Tgelgm8slCX07nbu7OxUrVqRixYoMGDAAd3d3AH755RdeeeUVOnfuTEREBGFhYfz6669ERkYyYsQITpw4wYwZM/DwSFen+iulHkLx4sXp378/Y8eOJSQk5JHXl5Rs88CaocaYi1iVisQ2oHzUurmUhOQcGxvLyy+/zOXLl2nTpg3dunXj7bffBmy/eEaMGMGwYcO4ePEiCxYs4LHHHnNm2EqpVCwoKIhx48Zx9OhRihQp8mgru9euu/l3mMQD+BUoAngCewC/O5bJAXha9zsCXz5oveXLl3/gz5jULCoqyrRr18489thjBjCvvvqquXTp0q32iRMnGsBUrVrVREdHOzFSpVxbahhySS4PO+TywCN4xpg44B1gLfAzsMgYs19EgkQkyFqsFLBfRH4B6gM9Hu1rJvXz8/Nj1qxZxMTEMGTIEJYtW4b9dMA9evRgwYIF7N27l7JlyxIcHKyVk5RSycql50NPSSdPnqRQIdvIlP1Pp5MnT9KxY0fWrl1LzZo1+eKLLx79Z5VS6hadD/1feo6dgyQk882bN1OsWDF69OjB1atXKVSoEKGhocyYMYPw8HCefvppPv/8c91bV8qBnLVjmpz+yzZpQnewSpUq0bVrVyZNmkSFChXYt28fIkKHDh2IioqiSpUqdO3aleeee4758+dz9epVZ4esVJqWMWNGzpw541JJ3RjDmTNnyJgx40O9TodckkloaCjt2rXj8uXLhISE0LhxY8D2h5o9ezYffvgh0dHReHt70759e/r27Yu3t7eTo1Yq7blx4wYxMTEut3OUMWNGChYsSIYMGW57Pt3WFHW2P/74gyZNmtC6dWvefffd29pu3rzJpk2bmDZtGkuXLiVr1qz07duXnj17kiVLFidFrJRK7TShO9H169fJkCEDIsKqVasoWbIkvr6+ty2zf/9+Bg4cyIoVKyhatCgRERG36qEqpZQ9PSjqRJ6enogIcXFxdO7cmRIlStCqVSt27959axk/Pz+WL1/O2rVrOXr0KD179nRewEqpNEsTegrx8PAgPDyc3r17s3r1ap555hmqVavG9u3bby1Tp04d+vfvz5w5c1i5cqUTo1VKpUWa0FNQvnz5GDt2LCdOnGD8+PHExMTcOn0xYehryJAh+Pv706lTJ06fPu3McJVSaYyOoTtRfHw8bm5uiAhdu3Yld+7cDBw4kAMHDhAYGEjTpk1ZtGiRzreulLpFx9BTKXd3d0SE+Ph4Ll26xLBhw6hSpQpXr17lgw8+YMmSJfTv39/ZYSql0ghN6KmAu7s7X375JSEhIRw/fpxKlSqxb98+3nzzTcaOHcuYMWOcHaJSKg1I0mTdIlIP+BRwx1bsYswd7V7APOBJa53jjDGzHRyry2vVqhUNGzZkzJgxTJs2jcjISG7cuEH//v3JkiUL77zzjg6/KKXu6YFj6FZN0UPY1RQFXjN2NUVFZADgZYzpJyK5gINAXmPM9XutV8fQ7++ff/4hS5YsXL9+nWLFinHixAmKFy9Op06dePPNN3niiSecHaJSygkedQz9Vk1RK0En1BS1Z4BsVnGLrMBZEqlYpJIu4WrRU6dO3dor//333+nduzc+Pj78+OOPzgxPKZUKJSWhFwBO2j2OsZ6zNxnbnOi/YytB18MYc9d0giLSSUTCRSTcvvK1urcCBQqwf/9+Jk2aRP78+QG4evUq9erVY+/evU6OTimVmiQloSelpmhdIBLIDwQAk0Uk+10vMma6MSbQGBOYK1euhww1/cqSJQvdunXj559/Zs2aNTRq1Ijs2bNTp04dJkyYwK+//ursEJVSqUBSEvoDa4piqye6zKqQdARbPdGSjglRJXBzc6Nu3bosX76c9evXExcXR79+/ShbtixTp051udnmlFIPJykJPQwoJiJFRMQTaAXceV36CaAWgIjkAUpgq0OqkknJkiVZt24d/v7+/PPPP7z99tt4e3uzZs0aZ4emlHISR9UUHQk8JyL7gA1AP2OMXreezMqVK0dYWBi//PILPj4+xMbG0rBhQxYtWuTs0JRSTqCX/ruI8+fP0759e7Zt28bp06eZNm0aHTp0cHZYSikH00v/04EcOXKwdOlSDh8+TO3atenYsSNVq1blypUrzg5NKZVCNKG7mOzZs7Ny5UqeeeYZfvjhB/LkycPmzZudHZZSKgVoQndBnp6eRERE0KVLFy5dukTNmjUZNGiQSxXRVUrdTRO6CwsODiY0NJQMGTIwatQogoODnR2SUioZaUJ3cfXq1WPLli14enoydepUzp49y40bN5wdllIqGWhCTwcqV67M6tWrOXz4MHXq1KFYsWJMmDCB+Ph4Z4emlHIgTejpRK1atVi0aBGRkZHExMTQq1cvqlSpwtq1azWxK+UiNKGnI02bNuXAgQM0a9YMgJ07d1KvXj18fHy4dOmSk6NTSj0qTejpTPHixVm8eDFhYWHUqFEDgIsXL/LNN99w8+ZNFi9erMWplUqjNKGnU4GBgWzcuJF169ZRrFgx3nzzTerUqcMbb7xBkSJFmDx5sp7mqFQak6SELiL1ROSgiBwRkfcTae8jIpHWLUpE4kXkcceHqxztxRdfZOfOnQQHB7Nlyxby5s2Lv78/3bp1o2HDhvz555/ODlEplUQPTOhWCbpgoD5QGnhNRErbL2OM+dgYE2CMCQD6A1uMMWeTIV6VDNzc3OjSpQubNm3i2rVrREZG0rZtWzZt2kTt2rV1T12pNMJRJejsvQaEOCI4lbKqVq1KeHg4/v7+fPnllzz77LM0b94cEcEYQ2BgIKNHjyYuTqsLKpUaOaoEHQAikhmoByx99NCUMxQsWJCtW7cybtw4tm/fzqRJkwgNDeWff/4hb968DBgwgEqVKmn5O6VSIUeVoEvQGPjhXsMtWlM0bXB3d+e9994jMjKSwoUL06RJE1atWsWqVatYvHgxJ0+eJDAwUKcSUCqVcVQJugStuM9wi9YUTVtKlizJ5s2bee6552jdujXTpk3jlVde4cCBA9SrV4/p06dr2TulUhFHlaBDRLyA6sAKx4aonMnLy4s1a9bQoEEDgoKC6NOnD1mzZmXZsmVs3LiRjBkzcv36dT1wqlQq4KgSdADNgO+MMf8kT6jKWTJlysQ333xDUFAQ48aN45lnnmHXrl14e3sTFxdHy5YtqVmzJgsWLNCCGko5UZLOQzfGrDbGFDfGFDXGjLKem2qMmWq3zBxjTKvkClQ5V4YMGZgyZQpr1qzh0qVLVK5cmW7dunHmzBlq1KjBsWPHeP3118mbNy8jRozQ+WGUcgK9UlQ9lLp16xIVFUXnzp2ZMmUKRYsW5dy5c+zZs4cNGzZQq1Ythg4dymeffebsUJVKdzShq4fm5eXF559/zoEDB2jQoAEjRoygbNmyxMfHs2zZMpYsWUJQkG007sKFC06OVqn0QxO6+s+KFy/OokWL+PHHH8mcOTN16tShc+fO1K5dm4wZMxIbG0u5cuV44403+Ouvv5wdrlIuTxO6emSVK1dm165d9OnTh5kzZ1KhQgV+/vln3NzcaNOmDYsXL6ZEiRIMHjyYDRs26IFTpZKJOOt0s8DAQBMeHu6UvlXy2bp1Ky1atODKlSt89dVXNG3alEOHDtGjRw/Wrl2LMYZdu3ZRrlw5Z4eqVJokIhHGmMDE2nQPXTlUtWrVCA8Pp0SJErz00kv079+fwoULExoayrlz51i9ejVly5YFoEePHowYMYJr1645OWqlXIMmdOVwhQoV4vvvv6dDhw6MGTOGwMBAIiIi8PLyon79+ri7uxMfH8+pU6cYOnQoRYsWZeDAgRw+fNjZoSuVpmlCV8kiY8aMzJgxg1WrVnH27FkqVapE9+7d+eWXXwDbfDELFixg3bp1+Pv7M2bMGIoXL86IESOcHLlSaZeOoatkd+7cOfr27cvcuXO5ceMG1apVY8CAAdStW/fWMr///juLFy+mXbt2ZM+e3YnRKpW66Ri6cqqcOXMyY8YMYmJiGDt2LDExMTRo0ICZM2feWiZ//vz06NGD7NmzExsbS6dOnfRUR6UekiZ0lWJy585N37592bdvH3Xq1KFjx46MHz/+ruX279/P/PnzqVSpEqtXr3ZCpEqlTQ6pKWotU8OqKbpfRLY4NkzlSjJnzsyKFSto2bIlvXv3pn///ty8efNWe4UKFdi0aROZMmWiYcOGNG/enJMnT95njUopAI8HLWBXU7Q2trnRw0RkpTHmgN0yOYDPgXrGmBMikjuZ4lUuwtPTkwULFpAzZ07GjBnD/v37+eqrr/Dy8gKgYsWK7Nmzh3HjxjFy5EhiY2MJDQ11ctRKpW6OqinaGlhmjDkBYIz527FhKlfk7u7OlClTmDx5MqGhoVSsWJEDB27tJ+Dp6cmAAQM4cOAAEyZMAGwHT0NCQnT+daUS4aiaosWBnCKyWUQiRKRtYivSEnTqTiJC165d2bBhA+fPn+fpp5+mcePGrFq16tYUvEWKFKFkyZIATJs2jdatW9O4cWN+++03Z4auVKrjqJqiHkB5oCFQFxgsIsXvepGWoFP3UK1aNSIjI3n//fcJDw+ncePG+Pj4MHz4cGJiYm4tN2TIECZOnMjGjRvx8/Pjiy++0LnXlbI4qqZoDLDGGPOPMeY0sBXwd0yIKr3Ily8fo0aN4sSJEyxduhQ/Pz+GDx9O4cKFadasGfv27cPd3Z0ePXqwd+9eypYtS4cOHRg5cqSzQ1cqVXBUTdEVwPMi4iEimYFK2MrVKfXQMmTIwMsvv8yaNWuIjo6mX79+bNq0CX9/f9q2bcuxY8fw9fVl8+bNfP3113To0AGAvXv3snXrVidHr5QTGWMeeAMaAIeAaGCg9VwQEGS3TB/gABAF9HzQOsuXL2+USqozZ86YPn36mIwZM5ps2bKZPXv23LVMixYtDGCaN29uoqOjnRClUskPCDf3yKt66b9KU44ePUrVqlXJkCEDYWFh2B+LiY2NZfz48YwZM4a4uDh69OjBwIEDb50KqZQr0Ev/lcsoUqQIy5cv56+//uLll1++berdzJkzM3jwYA4fPsxrr73Gxx9/zNSpU++zNqVciyZ0leZUqFCBOXPmsG3bNjp06HBXBaT8+fMzZ84cIiIi6N69OwCRkZGcOXPGGeEqlWI0oas06dVXX2X48OHMmzePYsWKMXPmTOLi4m5b5plnniFTpkzcuHGDl19+mXLlyrFjxw4nRaxU8tOErtKsIUOGsHnzZgoWLEjHjh3x9/dn//79dy2XIUMGFi1ahLu7O88//zxvv/22XpSkXJImdJWmVa9ene3bt7Ns2bJbhTSWLl1613KBgYHs3r2bTp068cUXX+Dr68vBgwedELFSyUcTukrzRIRmzZoRERFBmTJleOWVVxg0aNBd873kyJGD4OBgDh48yMCBAyle3HYx888//6xzwyiXoAlduYz8+fOzZcsW2rdvz6hRo+jTp0+iibpIkSIMGjQIESEmJobAwEBq1apFVFSUE6JWynE0oSuX8thjjzFjxgy6devG+PHj+fDDD++7fL58+Rg/fjy7d+/G39+ft99+G504TqVVmtCVyxERJk6cSJs2bRg0aBCffvop169fT3RZd3d3goKCOHLkCF27dmXGjBkUL15cT3FUaZImdOWS3NzcmDVrFk2bNqVnz55kypQJHx8f6taty+rVq+8aivH29mbSpEns27ePDz74AG9vbwBOnDjhjPCV+k80oSuX5eHhwddff838+fMZNGgQzz//PIcOHaJhw4bUqFGDbdu23TX1bqlSpejatSsAO3fu5KmnnmLAgAF3neOuVKp0r0lezO2Tc9UDDgJHgPcTaa8BXAAirduQB61TJ+dSznDt2jUTHBxs8uTJYwCTIUMGU6xYMdO4cWNz+PDh25a9cOGCad++vQFMtWrVzG+//eakqJX6F48yOZdVU/QQdjVFgdfM7TVFawC9jTGNkvpFopNzKWe6fPkyixcv5tChQ/z6669899135M6dmx07dpAzZ87blv3qq68ICgoia9asfPXVV9SpU8dJUSv16JNzJaWmqFJpStasWWnXrh2jR4/m66+/ZuXKlRw9epSWLVveNbzSpk2bWzM77t2710kRK/VgjqopClBZRPaISKiI+CW2Iq0pqlKr559/nqlTp7J+/Xp69ep1V3vp0qUJDw+/1bZu3Tr27NmT0mEqdV+Oqim6CyhsjPEHPgOWJ7YiozVFVSr2v//9j169evHZZ5/RokULDh06dFt7xowZcXNz4+bNm7z77rsEBgYyfPjwe54SqVRKc0hNUWPMRWPMZev+aiCDiDzhsCiVSiEfffQRw4YNIzQ0lNKlS9O5c2eOHDly2zJubm5s2bKFV199lWHDhlGhQgV27drlpIiV+pdDaoqKSF4REet+RWu9emWGSnPc3d0ZOnQo0dHRvP3228yePZtixYpRu3ZtFi9efGt83dvbm3nz5rFixQr+/vtvKlasmOhMj0qlpAcmdGNMHPAOsBZb4edFxpj9IhIkIkHWYq8AUSKyB5gEtDIPOn1GqVQsT548fPbZZxw9epQRI0Zw6NAhWrZsSeXKlW9L3E2aNOHAgQN8+umn+PnZDh1dvXrVWWGrdE5riiqVBPHx8SxatIju3btz8eJFhg0bRp8+ffDw8Lhtud27d9OoUSPmzp3Liy++6KRolSvTmqJKPSJ3d3dee+019u/fT5MmTRgwYAC1a9e+ayKv3Llz4+3tTYMGDQgJCXFStCq90oSu1EPInTs3ixcvZu7cuWzfvp3AwMDbDogWKFCArVu38txzz9G6dWt69erFH3/84cSIVXqiCV2p/6Bt27Zs27aNmzdvUqVKFdq0acOwYcP48ssviYuLY82aNbRv355PP/2UBQsWODtclU7oGLpSj+Dvv/+ma9eu/PTTT8TExGCMoUCBAixbtoyKFSsSHR1Nnjx5yJo1K8uWLePgwYO89957eHp6Ojt0lUbpGLpSySRhCObEiRNcuXKFbdu2kSFDBqpVq8bs2bMpWrQoWbNmBWDDhg0MGDCAgIAAvv/+eydHrlyRJnSlHOSxxx6jSpUqhIWFUbVqVf73v//RunVrjh49CkBwcDCrVq3iypUrVKtWjW7dunH58mUnR61ciSZ0pRzsiSeeYM2aNQwZMoRvvvmGEiVK0KNHD06dOkXDhg2Jioqie/fuBAcHs3btWmeHq1yIjqErlYx+++03hg8fzqxZs8iZMyfBwcG0bNkSgKioKPz8/BARIiMjKVu2LG5uuo+l7k/H0JVykgIFCjB9+nT27NlDkSJFePXVV2nRogWnTp2iTJkyiAgnTpygcuXK1K9fXwtUq0eiCV2pFODn58ePP/7I6NGjWblyJRUrVrw16VehQoWYNGkSW7ZsISAggG3btjk5WpVWaUJXKoV4eHjw/vvv88MPP3Dp0iWqVq3Kvn37EBE6duzIjh07yJw5MzVq1OCjjz5ydrgqDUpSQheReiJyUESOiMj791mugojEi8grjgtRKdcSGBjI1q1b8fDwoHr16vz4448ABAQEEB4eTrNmzfTqUvWfPDChWzVFg4H6QGngNREpfY/lxmKblVEpdR+lS5dm27ZteHt7U61aNQYPHsz169fx8vJi0aJFfPzxxwAcPXqUa9euOTlalVY4sqZoN2Ap8LcD41PKZfn4+LBz507eeOMNPvjgAwIDAwkPD0dE8PDwuDUs06xZM52SVyWJQ2qKikgBoBkw9X4r0pqiSt0uZ86czJkzh1WrVnHmzBkqVKjAa6+9xpEjR8iWLdut6klNmzbVi5DUAzmqpuhEoJ8xJv5+K9KaokolrmHDhhw4cICBAweycuVKSpUqRZcuXWjcuDGzZs1i/fr1vPDCC3pao7ovh9QUBQKBhSJyDFv1os9F5CVHBKhUeuHl5cUHH3xAdHQ0nTp1YsaMGRQtWpTo6GjmzZvHvn37GDhwoLPDVKnYA68UFREP4BBQC/gNW43R1saYRAsoisgcYJUxZsn91qtXiip1f0eOHGHw4MEsXLiQxx9/nDZt2jBw4EBy5crF9evXdcbGdOqRrhRNYk1RpZSD+fr6EhISQkREBIGBgXz66aeUL1+e6dOnU6VKFYYOHUp8/H1HOVU6o3O5KJVGbNy4kffff5+wsDC8vLy4cOEC1atXZ/LkyZQpU8bZ4akUonO5KOUCXnjhBX766SeWLFlCnjx5ANi2bRv+/v4EBQXpWTBKE7pSaYmI0Lx5c/bv38+0adPIlSsXN2/eJCQkhP379+OsX9wqddCErlQa5OHhQadOnYiOjmbMmDGICM8++yz58+enYMGCfPTRR5rc0yFN6EqlYZkzZ6Zfv34cPXqU6dOn4+fnx++//06/fv3InTs3kydP5sKFC84OU6UQPSiqlIs5e/Ysr7/+OmvWrAEgW7ZsHDhwgIIFCzo5MuUIelBUqXTk8ccfJzQ0lBUrVpAlSxbi4uKYNGkShw8fdnZoKplpQlfKRTVp0oSIiAjq16/PJ598QvHixSlcuDCbN292dmgqmWhCV8qFlShRgqVLl3Ly5ElefPFFTpw4Qc2aNalduzZRUVHODk85mCZ0pdKBfPnysW7dOmbPno2npycbNmygbNmyjBs3Ts+GcSGa0JVKR9566y327t1LqVKlMMbQp08fXn/9dWJjY50dmnIAh5SgE5GmIrJXRCKt+c6rOj5UpZQjlChRgvDwcD766CNGjhzJwoULqVKlCseOHXN2aOoRJWW2RXdssy3WxjaVbhjwmjHmgN0yWYF/jDFGRMpim8Cr5P3Wq6ctKpU6zJ8/n3bt2pElSxaWLFlCrVq1nB2Suo9HPW3xgSXojDGXzb/fDFm4uwCGUiqV8vLywt3dnfPnz/Piiy/Sq1cvHVdPoxxSgg5ARJqJyC/At8D/HBOeUiq5NWrUiOPHj9O7d28yZMjAhAkTyJ8/P5cuXXJ2aOohOaoEHcaYb6xhlpeAkYmuSGuKKpUq5c6dm48//pi//vqLevXq8eeff1K9enWOHz/Ot99+y7Vr15wdokoCR5Wgu8UYsxUoKiJPJNKmNUWVSsVy5sxJaGgoq1atIjo6moCAABo1akSJEiWYNWsWN27ccHaI6j6SktDDgGIiUkREPIFWwEr7BUTEV0TEuv8M4AmccXSwSqmU0bBhQ8LCwvD19QXgr7/+on379vj4+DBy5Eid8CuVclQJuuZAlIhEAsHAq0aPqiiVphUvXpydO3eyePFiChWy/UiPjY1lzJgxTo5M3UuSzkM3xqw2xhQ3xhQ1xoyynptqjJlq3R9rjPEzxgQYYyobY7YlZ9BKqZQhIrzyyiscOHCA6dOnkzlzZmJjY3n99deJjIyke/fuREdHOztMZdHpc5VSSRYbG8tnn33GmDFjOH/+PBkyZMDT05OQkBAaN27s7PDSBZ0+VynlEAkFNaKjo+nbty9ubm7ExsbSrFkzli9f7uzw0j1N6Eqph/b4448zduxYjhw5QuPGjYmPj+fll1/m66+/dnZo6ZomdKXUf1awYEGWL1/OsGHDMMYwaNAgjh8/ztmzZ/VqUyfQhK6UeiQiwtChQwkNDeXUqVPUqlWLl156CR8fH+bOnevs8NIVTehKKYeoV68ea9as4c8//+To0aPkzZuXt956i3feeUcvSEohmtCVUg7z7LPPsmLFCv7++28AunfvTnBwMLVq1eLixYtOjs71aUJXSjlUrVq1WLhwIREREXz33XcMGzaMy5cvkzlzZgDi4+OdHKHr0oSulHK4Zs2asWbNGv755x9GjBjBCy+8QFxcHJcvX6ZUqVIMHTqUM2d0dhBH04SulEoWL774IlFRUbRv357x48dTokQJZsyYgZ+fHyNGjODJJ5/k3Xff5c8//3R2qC5DE7pSKtlkz56d6dOns2nTJnLnzk2vXr04fPgwc+fOpXnz5nz22WdUqlTp1pi7ejSOqin6ulVTdK+I/Cgi/o4PVSmVVtWoUYOdO3eyaNEirly5QqdOnWjevDk7d+6kVatW6HTajvHAhG7VFA0G6gOlgddEpPQdix0FqhtjymIrbjHd0YEqpdI2EaFFixbs3LkTf39/mjdvzv79+xk7diwiQnR0NL/88ouzw0zTHFVT9EdjzDnr4Q5sRTCUUuou3t7erF+/nurVq9O2bVsmT56MMYYWLVpQpkwZWrRowZYtW/RK0//AYTVF7bQHQhNr0BJ0SimAbNmy8e2339K0aVO6devGqFGjWLFiBe+99x4bN26kRo0aVKtWjaNHjzo71DTFYTVFAUSkJraE3i+xdi1Bp5RKkDFjRpYsWUKbNm0YPHgwn376KWPGjOHkyZNMmTKFvXv38t133zk7zDTFIwnLJKmmqIiUBWYC9Y0xeoKpUuqBPDw8mDNnDl5eXowfP56oqCh69OhBx44deemll8iTJw8AM2fOxNPTkyZNmpAjRw7nBp2KOaqm6JPAMqCNMeaQ48NUSrkqNzc3Jk2axMcff8zu3btp0KABTz31FPPmzePq1asAzJo1izfffJPcuXPTsWNH4uLinBx16uSomqJDAG/gcxGJFBEtRaSUSjIRoXfv3pw8eZJFixZRtGhR+vTpQ7FixZg5cyZbtmxhx44ddOjQgZkzZ/Lmm2/qFAKJ0BJ0SqlUacuWLfTr14+ffvqJIkWK0LFjR9q1a8fcuXP54IMP2LlzJ6VKlXJ2mClOS9AppdKc6tWrs337dpYvX46Pjw8DBgygUKFCHDp0iIiICEqVKsXNmze5cuWKs0NNNTShK6VSLRGhadOmbNy4kYMHD9K1a1dmz55NixYtOHr0KMHBwQQEBPDjjz86O9RUQRO6UipNKF68OBMnTiQ0NJQTJ04QGBjI5cuXuXr1KlWrVqVPnz7p/mCpJnSlVJpSt25dwsLCyJcvHwMGDKBEiRI0b96ccePG0bhx43RdSEMTulIqzfH19SUsLIyxY8cSFhbG0qVL8fPzY926dQwZMoTIyEhnh+gUmtCVUmlSpkyZ6Nu3L7/++iv9+vUjPj4ed3d3Pv30U8qXL8/kyZOdHWKK04SulErTcubMyejRo/n555+5cuUKJ06cICAggG7duhEUFPTgFbgQTehKKZfh5uZGoUKFWLJkCdmzZ2fatGk0btz41hWnrk4TulLK5RQpUoRff/2VvHnzsmrVKnLkyMGwYcNc/upSTehKKZfk7e3NsWPH6NKlC/Hx8QwfPpwyZcrQt29f1q1b55IXJGlCV0q5rMcee4zg4GDOnj3L/PnzyZMnD5988gl16tShYMGCLlchyVE1RUuKyHYRuSYivR0fplJK/XfZsmWjdevW/N///R85cuTA29ubmzdv0qJFC2JjY50dnsM4qqboWaA7MM7hESqllINky5aNDRs24OnpyY0bN27Nv+4qHFVT9G9jTBhwIxliVEoph/H392f79u2UKFECsBXPmDdvnpOjcozkqCl6T1pTVCmVGhQuXJgffviBoKAgChYsSFBQEKGhiZZCTlMcWlP0QbSmqFIqtciYMSNTpkxhx44d+Pr60qBBAxo1apSmJ/hKSkJPUk1RpZRKiwoUKMAPP/xA0aJF+fbbb8mTJw8TJkxg69at/PXXX84O76E4pKaoUkqlZVmyZOHQoUO8+uqrnD17ll69elG9enUKFCjAokWLnB1ekjmkpqiI5BWRGKAXMEhEYkQke3IGrpRSjuTm5sbChQvZvn07jz/+ONmzZ8ff358333yTtFIuU2uKKqXUHY4cOcL06dPp1asXlSpVIi4ujrCwMPLnz+/s0LSmqFJKPQxfX18++ugj8ubNy+eff87ff/9Nw4YNOX36tLNDuy9N6EopdR8XLlzAGENkZCSFCxdm5MiRXL582dlhJUoTulJK3Ufr1q3ZuXMnBQoU4Nq1awwZMgRfX99UWZhaE7pSSj3AM888Q1hYGGXKlMHd3R0PDw8aNGiQ6krdaUJXSqkkyJcvH1u2bKFHjx5s2LCBbNmyUatWrVQ1Y6MmdKWUSiIvLy/Gjx9PiRIl+Pbbbzl37hxPP/00/fv3TxVXmGpCV0qp/6B06dKMGjUKgDFjxpA1a1a6dOnCpUuXnBaTJnSllPoPPDw86N+/PxcuXKBz584YY5gyZQq+vr6sX7/eKTFpQldKqUeQOXNmpk6dypkzZxgxYgQ5cuSgdu3avP322yl+eqMmdKWUcoCsWbMyePBgIiMjadu2LVOnTqVGjRqcPXs2xWLQhK6UUg6UKVMmatWqhZubGxEREZQvX57ff0+ZCWodVVNURGSS1b5XRJ5xfKhKKZU2tG3blk2bNpE/f36OHTtGqVKl2L59e7L366iaovWBYtatEzDFwXEqpVSaUq1aNQ4fPkyrVq24ePEiVapUoV27dpw4cSLZ+nRITVHr8ZfGZgeQQ0TyOThWpZRKUzJnzkxISAgbN27knXfeISQkBF9fX8aNG5cs/TmqpmiS6o5qTVGlVHpUs2ZNJk2axKFDh2jQoAG+vr7J0o9HEpZJSk3RJNUdNcZMB6aDbT70JPStlFIu48knn2T58uXJtn5H1RTVuqNKKeVkjqopuhJoa53t8ixwwRjzh4NjVUopdR8PHHIxxsSJSEJNUXdgVkJNUat9KrAaaAAcAWKBdskXslJKqcQkZQwdY8xqbEnb/rmpdvcN0NWxoSmllHoYeqWoUkq5CE3oSinlIjShK6WUi9CErpRSLkJsxzOd0LHIKeD4f3z5E8BpB4aTFvrWbU4ffae3fp3Zd1rd5sLGmFyJNTgtoT8KEQk3xgSmp751m9NH3+mtX2f27YrbrEMuSinlIjShK6WUi0irCX16Ouxbtzl99J3e+nVm3y63zWlyDF0ppdTd0uoeulJKqTtoQldKKReRqhO6s4pTJ6HfkiKyXUSuiUhvR/T5EH2/bm3rXhH5UUT8U6jfplafkVbVqaqO6DcpfdstV0FE4kXklZToV0RqiMgFa5sjRWSII/pNSt92/UeKyH4R2ZIS/YpIH7vtjbLe78dTqG8vEfk/EdljbbNDZm1NQr85ReQb6/O9U0TKOKjfWSLyt4hE3aPd8fnLGJMqb9im6o0GngI8gT1A6TuWaQCEYquY9CzwUwr1mxuoAIwCeqfwNj8H5LTu10/Bbc7Kv8dcygK/pNQ22y23Edusn6+k0DbXAFY56bOdAzgAPJnwmUup99pu+cbAxhTc5gHAWOt+LuAs4JkC/X4MDLXulwQ2OGibqwHPAFH3aHd4/krNe+jOKk79wH6NMX8bY8KAG4/Y13/p+0djzDnr4Q5s1aFSot/LxvoUAllIpMRgcvVt6QYsBf5O4X6TQ1L6bg0sM8acANtnLoX6tfcaEOKAfpPatwGyiYhg24E4C8SlQL+lgQ0AxphfAB8RyfOI/WKM2YptG+7F4fkrNSd0hxWnToZ+k8vD9t0e2zd8ivQrIs1E5BfgW+B/Dug3SX2LSAGgGTAVx0nqe13ZGgIIFRG/FOy7OJBTRDaLSISItE2hfgEQkcxAPWxfoo6QlL4nA6Wwla/cB/QwxtxMgX73AC8DiEhFoDCO2VFyRGwPJTUndIcVp06GfpNLkvsWkZrYEnq/lOrXGPONMaYk8BIw0gH9JrXviUA/Y0y8g/pMar+7sM2b4Q98BixPwb49gPJAQ6AuMFhEiqdAvwkaAz8YY+63h+novusCkUB+IACYLCLZU6DfMdi+PCOx/RLczaP/MkgKh+eaJFUschJnFad2ZsHrJPUtImWBmUB9Y8yZlOo3gTFmq4gUFZEnjDGPOrlRUvoOBBbafonzBNBAROKMMcuTs19jzEW7+6tF5PMU3OYY4LQx5h/gHxHZCvgDh5K53wStcNxwS1L7bgeMsYb2jojIUWxj2juTs1/r79wObAcqgaPWLbk5Ptc4YvA/OW7Yvmx+BYrw78EMvzuWacjtBxV2pkS/dssOw7EHRZOyzU9iq936XAr368u/B0WfAX5LeJxS77e1/Bwcc1A0Kduc126bKwInUmqbsQ09bLCWzQxEAWVS4r0GvLCN/WZJ4c/YFGCYdT+P9Rl7IgX6zYF18BXoiG1c21Hb7cO9D4o6Pn85KvDkuGE7CnwI21HqgdZzQUCQdV+AYKt9HxCYQv3mxfbtehE4b93PnkJ9zwTOYftpGgmEp1C//YD9Vp/bgaop9Xe+Y9k5OCChJ3Gb37G2eQ+2A9CO/BJ94DYDfbCd6RIF9EzBft8CFjpqWx/i/c4PfGf9X44C3kihfisDh4FfgGVYZ5E5oN8Q4A9sJ0/EYBsiTdb8pZf+K6WUi0jNB0WVUko9BE3oSinlIjShK6WUi9CErpRSLkITulJKuQhN6CpNEBFvu1kA/xSR36z750XkQDL0N0weciZNEbl8j+fnOGqGSKXuRxO6ShOMMWeMMQHGmABsc7pMsO4HAA+c70NEUvNV0Uo5hCZ05QrcRWSGNYf2dyKSCcCa2OpDay7xHiJSXkS2WJNdrU2Y2U5EuovIAWtO6oV26y1treNXEeme8KSI9LLmCo8SkZ53BmPNcz3ZWue32KZbTmgbY9fXuOR6Q1T6pHstyhUUA14zxnQUkUVAc2Ce1ZbDGFNdRDIAW4CmxphTIvIqtvns/we8DxQxxlwTkRx26y0J1ASyAQdFZAq2ueDbAZWwXen3k4hsMcbstntdM6AE8DS2S9gPALOsQhHNgJLGGHNHX0o9Mk3oyhUcNcZEWvcjsM2fkeBr698SQBlgnTXJlzu2y7IB9gLzRWQ5t8+o+K0x5hpwTUT+xpacqwLfGNukWYjIMuB5bDP0JagGhBjb7JC/i8hG6/mLwFVgprXnvuq/b7JSd9MhF+UKrtndj+f2HZV/rH8F2J8wDm+MedoYU8dqa4htTo3yQITdeHti601sytPEJDb9cBy2Sb6WYpuCeE0S16VUkmhCV+nFQSCXiFQGEJEMIuInIm5AIWPMJqAvtpn3st5nPVuBl0Qks4hkwTaE8n0iy7QSEXdrnL6m1WdWwMsYsxroie2ArlIOo0MuKl0wxly3Th2cJCJe2D77E7HNwjfPek6wnT1z3hqWSWw9u0RkDv/O0T3zjvFzgG+AF7DNoHcI29g92MbiV4hIRquvdx20eUoB6GyLSinlKnTIRSmlXIQmdKWUchGa0JVSykVoQldKKRehCV0ppVyEJnSllHIRmtCVUspF/D9vxY6vWPO4wAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(df_rand.threshold, df_rand.tpr, color='black', linestyle='solid', label='TPR')\n",
    "plt.plot(df_rand.threshold, df_rand.fpr, color='black', linestyle='dashed', label='FPR')\n",
    "plt.legend()\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "plt.yticks(np.linspace(0, 1, 11))\n",
    "\n",
    "plt.xlabel('Thresholds')\n",
    "plt.title('TPR and FPR for the random model')\n",
    "\n",
    "#plt.savefig('04_fpr_tpr_plot_random.svg')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ideal baseline:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>threshold</th>\n",
       "      <th>tp</th>\n",
       "      <th>fp</th>\n",
       "      <th>fn</th>\n",
       "      <th>tn</th>\n",
       "      <th>tpr</th>\n",
       "      <th>fpr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>486</td>\n",
       "      <td>1374</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.1</td>\n",
       "      <td>486</td>\n",
       "      <td>1188</td>\n",
       "      <td>0</td>\n",
       "      <td>186</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.864629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.2</td>\n",
       "      <td>486</td>\n",
       "      <td>1002</td>\n",
       "      <td>0</td>\n",
       "      <td>372</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.729258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.3</td>\n",
       "      <td>486</td>\n",
       "      <td>816</td>\n",
       "      <td>0</td>\n",
       "      <td>558</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.593886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.4</td>\n",
       "      <td>486</td>\n",
       "      <td>630</td>\n",
       "      <td>0</td>\n",
       "      <td>744</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.458515</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.5</td>\n",
       "      <td>486</td>\n",
       "      <td>444</td>\n",
       "      <td>0</td>\n",
       "      <td>930</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.323144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>0.6</td>\n",
       "      <td>486</td>\n",
       "      <td>258</td>\n",
       "      <td>0</td>\n",
       "      <td>1116</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.187773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.7</td>\n",
       "      <td>486</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>1302</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.052402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>0.8</td>\n",
       "      <td>372</td>\n",
       "      <td>0</td>\n",
       "      <td>114</td>\n",
       "      <td>1374</td>\n",
       "      <td>0.765432</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>0.9</td>\n",
       "      <td>186</td>\n",
       "      <td>0</td>\n",
       "      <td>300</td>\n",
       "      <td>1374</td>\n",
       "      <td>0.382716</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>485</td>\n",
       "      <td>1374</td>\n",
       "      <td>0.002058</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     threshold   tp    fp   fn    tn       tpr       fpr\n",
       "0          0.0  486  1374    0     0  1.000000  1.000000\n",
       "10         0.1  486  1188    0   186  1.000000  0.864629\n",
       "20         0.2  486  1002    0   372  1.000000  0.729258\n",
       "30         0.3  486   816    0   558  1.000000  0.593886\n",
       "40         0.4  486   630    0   744  1.000000  0.458515\n",
       "50         0.5  486   444    0   930  1.000000  0.323144\n",
       "60         0.6  486   258    0  1116  1.000000  0.187773\n",
       "70         0.7  486    72    0  1302  1.000000  0.052402\n",
       "80         0.8  372     0  114  1374  0.765432  0.000000\n",
       "90         0.9  186     0  300  1374  0.382716  0.000000\n",
       "100        1.0    1     0  485  1374  0.002058  0.000000"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_neg = (y_val == 0).sum()\n",
    "num_pos = (y_val == 1).sum()\n",
    "\n",
    "y_ideal = np.repeat([0, 1], [num_neg, num_pos])\n",
    "y_pred_ideal = np.linspace(0, 1, num_neg + num_pos)\n",
    "\n",
    "df_ideal = tpr_fpr_dataframe(y_ideal, y_pred_ideal)\n",
    "df_ideal[::10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA9TElEQVR4nO3deXwUVbbA8d9JCBBIICwiO0EwLAKyDyBLBBEUEJBFEAlpluA4bu+NziDquI6MM4z7yCJLQHwgIijKIgqGHR0ZUUAgIosE3Agga5DAfX90hWlClg6p7urlfD+f/tDdt7ruqU5zuvpW1T1ijEEppVTwi3A6AKWUUvbQhK6UUiFCE7pSSoUITehKKRUiNKErpVSI0ISulFIhQhO6spWI7BORmxyOIVpEPhCRX0XkHR/18aSIzLFpXZNF5PEC2o2I1LehnzQRGV3c9VxBv4kikuHlsra9r+FIE3oAEpGTHrcLInLG4/Ew60N/znp8TEQ2iEh767XJInLeajsuIl+JSG+ntwlARFJF5Ldc23eH1bbPYzt/EpGZIhJjtaWJSJbVdlhEFopItQK6GghcDVQyxgyyIW6vE9KVMMbcbYx5xlfrV+FDE3oAMsbE5NyA74E+Hs+9ZS32ttV+FbAOWCgiYrVttNrigNeBeSIS59+tyNffPbfPGPO2R1sfK+6WQBvgMY+2e622+kAMMLGAPuoA6caY7KIGJyIlivoapQKFJvQgZ4w5B8wCqgKVcrVdAN4EygLX5vV6EakgIh+KyC8ictS6X9OjPU1EnhGR9SJyQkRWiEhlj/bhIrJfRDJF5FGbtukgsAxokkfbMeA9oHk+2/MU8BfgDmuPfpSIRIjIY1acP4vIbBEpby0fbw1pjBKR74FVudZX1oqluseviupWc0lrXSdEZLuItPZ4XXURedd6X/eKyP35ba/1y+VZj8cPi8gPInJIREbmWraUiEwUke+tXzKTRSTaaivwb1kQ61ffOyIyx9qerSKSICKPWO/ZARG5Odf2LRaRIyKyW0TGeLRFW9t0VES+wf3lTK7XevXeqKLRhB7kRKQUkAxkGGMO52qLBFzAOWB/PquIAGbi3qutDZwBXsu1zJ3WeqoAJYGHrPU3BiYBw4HquL9QvEoghWxTLeBW4Ms82ioBtwO783qtMeYJ4DmsXzDGmOm4359k4EbgGtx7+Lm3sQvQCOiRa32ngFuAQx6/Kg5ZzbcB83D/Elqcs04RiQA+AL4CagDdgAdF5JJ157PtPXG/v91xfwnnPh7xPJCA+wutvrX+v1ht3vwtC9IH9w5ABdzv/UfWOmsATwNTPJadC2Tg/rsPBJ4TkW5W2xNAPevWAxjhsX1X/N4oLxhj9BbAN2AfcFOu554EfgOOAT/j3qtsZbUlA9lW2znc/6kHF6G/5sBRj8dpwGMej+8Bllv3/wLM82gra8V1Uz7rTgWyrNiOAYdzbedJ6/n9uIeKoj1iOA38ChhgC1C7gG14Epjj8XglcI/H4wbWe1MCiLfWeU0B60vE/YWZu49PPB43Bs5Y938HfJ9r+UeAmQW8L89a92cAf/NoS7Diqw8IcAqo59HeHthbhL/l6ALes489Hvex/h6R1uNYK444oBZwHoj1WH4CkGrd3wP09GhLyXn/Cntvcv/t9Fa0m44XBq/5xpi78mnbZIzpaB1UnA50AubntaCIlAFeBHri3jMDiBWRSGPMeevxjx4vOY17Dxfce2cHchqMMadEJLOQuCcaYx7Lp62fMeaTfNruN8ZME5GmwIe4fwl8X0hfOapz6S+U/biT+dUezx2g6HK/L6WtMfg6uIdojnm0RwJrvYx1c65Yc1wFlAE2//dwCWKt29u/ZUF+8rh/BvcX7nmPx+D+21cHjhhjTuSKM2fI6ZLPRa5tKM57owqhQy4hzBhzEvce9XARaZHPYn/Evcf6O2NMOaCz9bzks7ynH3Dvrblf4E4olfJfvPiMMVuBZ4F/eRwELswh3IkkR23cv2I8E1hB044WdUrSA7j3muM8brHGmFu9eO0l76kVa47DuBPrdR7rLW/cB4uheH/LojgEVBSR2FxxHvRiG4rz3qhCaEIPccaYTGAa/x1nzS0Wd5I4JiIVcY9/emsB0FtEOopISdzjrP74TM3CPZ5/m5fLzwX+R0TqWr9acsbYvT0L5iegUs6BVC98DhwXkT9bBwgjRaSJiLQp9JXuX1LJItLY+oK8+Pcw7oPcbwAvikgVABGp4TH+XJy/pdeMMQeADcAEESktIs2AUUDOGVjzgUesg7Q1gfs8Xl6c90YVQhN6eHgJuNX6j5dXWzTuvb9NwHJvV2qM2Q78Afg/3HtlR3EfKPMpY8xvwCtAvhfj5DID98G+NcBe3OP49xX4ikv724n7S2GPuM/7r17I8udxj0E3t/o7jPtLtdAvBGPMMtx/k1W4D/yuyrXIn63nN4nIceAT3HvlUIy/5RUYivv4wyFgEfCEMeZjq+0p3MMse4EVuN97oHjvjSqcWAcilFJKBTndQ1dKqRChCV0ppUKEJnSllAoRmtCVUipEOHZhUeXKlU18fLxT3SuliuHEiRPExsYWvqCy3ebNmw8bY67Kq82xhB4fH88XX3zhVPdKqWI4cOAAtWrVKnxBZTsRyW9eJh1yUUoVXWZmYTM8KCcUmtBFZIY1fea2fNpFRF6xptD8WkRa2h+mUiqQHDt2zOkQVB682UNPxT3ZT35uwT3N57W4Z1WbVPywlFKBrFWrVk6HoPJQ6Bi6MWaNiMQXsEhfYLZxX3K6SUTiRKSaMeYHu4L0dPjwYXbs2IH38zIppewQGRlJ06ZNiYmJYfPmzSQmJjodEgDnzp0jIyODrKwsp0OxVenSpalZsyZRUVFev8aOg6I1uHSqzAzrOZ8k9FWrVnHHHXf4YtVKqUJERUXRsWNHrr/+etq1a0fp0qWdDomMjAxiY2OJj48PmR09YwyZmZlkZGRQt25dr19nR0LP6x3Mc4IYEUnBPSxD7dq181qkUB07dqRnz54sX76cfv368fvf/56ICD22q5SvnTlzhjVr1vDRRx/x0ksvUb58eZ588kmnwyIrKyukkjmAiFCpUiV++eWXor3Om8m5rCGXD40xl9V4FJEpQJoxZq71eBeQWNiQS+vWrc2VnrZojOGhhx7ihRde4K677mLGjBlF+lmilCqeNm3a8Msvv7Bnzx7Hd6h27NhBo0aNHI3BV/LaNhHZbIxpndfydvwlFgNJ1tku7YBffTV+nkNEmDhxIs8++yxz5szhrrvyK9yjlPKFkSNHsn//fj799FOnQ1EeCh1yEZG5uGsqVhaRDNyT5kcBGGMmA0txF/TdjbsMl8tXweaKi0cffZQKFSpQo0YNf3SplLI0a9aMuLg4Zs6cSbdu3Qp/QQjLzMy8+B78+OOPREZGctVV7gs5v/rqK66//nqys7Np1KgRs2bNokyZMhcPMGdnZ1O3bl3efPNN4uLiih2LY/OhF2fIJT+LFy+mffv2F99MpZRvpKWlMX/+fGbOnMmPP/5I+fLO1acIpCGXJ598kpiYGB566CEAYmJiOHnyJADDhg2jVatW/O///u8lz48YMYKEhAQeffTRy9bnxJBLQDh8+DDDhg2jQ4cOfPfdd06Ho1RIa9WqFS6Xi6ysLObNm+d0OEGhU6dO7N69+7Ln27dvz8GDB/N4RdE5NpeL3SpXrsyKFSvo06cP7du358MPP6Rt27ZOh6VUSNq8eTNdunThuuuuY+bMmYwdO9bpkAB48MEH2bJli63rbN68OS+99FKx1pGdnc2yZcvo2fPSazTPnz/PypUrGTVqVLHWnyNk9tDB/U23YcMGYmNjSUxMZOnSpU6HpFRIqlSpEiKCy+Xis88+Y8eOHU6HFJDOnDlD8+bNad26NbVr176YuHOer1SpEkeOHKF79+629Bcye+g5EhIS2LhxI7fccgubN2/m1ltvdTokpUJOztS5d911F+PGjWPatGn885//dDgqir0nbbfo6Og8fzHkPP/rr7/Su3dv/vWvf3H//fcXu7+Q2kPPUaVKFdatW8djjz0GuMfXlVL22bdvHwBXX301/fr1Y9asWSF36b0/lC9fnldeeYWJEydy7ty5Yq8vJBM6uL8BRYTvvvuOBg0a8PTTT+PUGT1KhRrPMy9SUlLIzMxk0aJFDkYUvFq0aMH1119vy8HlkBtyya127dr07t2bJ554ggMHDjBp0iRKlAj5zVbKp/bt28fVV18NQLdu3bjmmmuYMmUKQ4cOdTgyZ+WeCiHn1MTccj//wQcf2NJ/yO6h54iKiiI1NZXHHnuMadOm0bdv33zfZKWUd86cOXPxfkREBGPGjGH16tXs2rXLwahUyCd0cF9V+swzzzBlyhSWL1/OhAkTnA5JqaCWez50l8tFiRIlmDp1qkMRKQiThJ4jJSWFlStX8vjjjzsdilJBbfPmzZc81oOjgSGsEjpAYmIipUuX5siRI/Tr14+9e/c6HZJSQadKlSqXPTd27FgyMzNZuHChAxEpCMOEnuP7779n7dq13HDDDWzfvt3pcJQKKiVLlrzsua5du1K3bl1mzJjhQEQKvEzoItJTRHZZhaDH5dFeQUQWWUWiPxeRy+ZNDzTNmzdn9erVAHTu3JnPPvvM4YiUCh4ZGRmXPRcREUFycjKrVq26eJ668q9CE7qIRAL/wl0MujEwVEQa51psPLDFGNMMSAJetjtQX2jSpAnr1q0jLi6Obt26sXbtWqdDUiooNGmS9z7biBEjAJg1a5Y/w3FcZGQkzZs3v3jbt28faWlplC9fnhYtWtCoUSOeeuopgEueb9iw4cWZGe3gzR56W2C3MWaPMeY3YB7uwtCeGgMrAYwxO4F4Ebnatih96JprrmHdunUkJiZSr149p8NRKiikp6fn+XydOnXo2rUrqampXLhwwc9ROSfnUv6cW3x8POCeYfHLL7/kiy++YM6cORcPJuc8/+WXX/Lhhx+yfv16W+LwJqHnVwTa01fA7QAi0haoA9TMvSIRSRGRL0Tki6LWyvOlatWq8eGHH1K9enWys7OZP3++XlWqVAHOnz+fb5vL5WLfvn0XhzQVlC1bllatWl02tXd0dDTNmzf36/S53hSB/hvwsohsAbYCXwLZl73ImKnAVHAXuChSpH7y1ltvkZyczOrVq3n11Vcdr5eoVCBq2rRpvm39+/enXLlyzJw5kxtvvNGPUbklJiZe9tzgwYO55557OH36dJ4T9iUnJ5OcnMzhw4cZOHDgJW1paWmF9pkzeyJA3bp1L5sGITMzk02bNvH4449fUvj56NGjfPvtt3Tu3LnwDfOCN9kqA6jl8bgmcMhzAWPMcWOMyxjTHPcY+lVAUJ4POHz4cB566CFef/11hg8fbsuEOUqFmoLmHC9TpgxDhw5lwYIFHD9+3H9BOchzyMUzma9du5YWLVpw8803M27cOK677rqLzzdr1oyqVavSu3dvqlatak8gxpgCb7j34vcAdYGSuIdXrsu1TBxQ0ro/Bphd2HpbtWplAtmECRMMYHr16mVOnTrldDhKBZSdO3cW2L5p0yYDmDfeeMPnsXzzzTc+76MwZcuWvey5Tz/91PTq1avA53ft2mWuuuoq8+WXX+a53ry2DfjC5JNXC91DN8ZkA/cCHwE7gPnGmO0icreI3G0t1gjYLiI7cZ8N84A9XzfOGTduHJMnT2b16tV8++23ToejVFBp27YtjRo1YubMmU6HEtASEhJ45JFHeP75521Zn1cDxMaYpcaYBGNMPWPMX63nJhtjJlv3NxpjrjXGNDTG3G6MOWpLdA4bO3Ys3333Hddffz0Ax44dczYgpQLEDz/8UGB7TjWjDRs2sHPnTj9FFZzuvvtu1qxZY8tV63rErxA5lzjPmDGDhg0bXjaHhVLhKOcAYEGGDx9OZGQkqampPo/HaXnN4JqYmMiHH35Y6PPR0dEcPHiQunXrFjsOTeheat++PaVKleLGG2/U07FU2Nu6dWuhy1StWpVbb72V2bNnk5192Ulvygc0oXupUaNGrF+/npo1a9KzZ888v3mVCheRkZFeLedyufjhhx9YsWKFjyNSoAm9SGrWrMmaNWto0qQJ/fv3Z8+ePU6HpJQjEhISvFquV69eVK5c2ecHR00IXgh4JdukCb2IKleuzMqVK3nzzTe55pprnA5HKUds27bNq+VKlizJXXfdxeLFi8nMzPRJLKVLlyYzMzOkkroxhszMTEqXLl2k14lTb0Lr1q3NF1984Ujfdlq7di0bNmzgT3/6EyJ5XVSrVOjZvXs39evX92rZrVu30qxZM15++WXuv/9+22M5d+4cGRkZIVdYo3Tp0tSsWZOoqKhLnheRzcaY1nm9RqslF9O8efN4/fXX+emnn5g4caJOFaDCwm+//eb1sk2bNqVNmzZMmzaN++67z/Ydn6ioKFvOEAkFmn2K6dVXX+WBBx7gxRdfZOjQoSG3l6BUXn7++eciLZ+SksLWrVvZtGmTjyJSoAm92CIiInjxxReZOHEi8+fP5+abb+bEiRNOh6WUT+UuEl2YIUOGEBsby5QpU3wUkQJN6LYQEf74xz8yd+5c4uPjKVu2rNMhKeVTRb3ALiYmhmHDhvH2229z9GhIXEgekDSh22jIkCHMnj2biIgIDhw4cNncx0qFiujo6CK/JiUlhaysLObMmeODiBTYV1O0vIh8ICJfich2EXHZH2rwMMYwdOhQOnbs6NUVdUoFm5yKPEXRokUL2rRpw9SpU0PqFMNAYldN0T8A3xhjrgcSgX+KyOVlwcOEiDB16lQiIyPp1KmTVxPkKxVMduzYcUWvS0lJYdu2bWzcuNHmiBTYV1PUALHiPh8pBjhCHhWLwknjxo1Zv3491atXp0ePHsybN8/pkJSyzZXsocN/D45OnTrV3oAUYF9N0ddwz4l+CHcJugeMMeFTITYfderUYf369bRr146JEyfqBEUqZFzpmVwxMTEMGTKEd955R88G8wFvEro3NUV7AFuA6kBz4DURKXfZigK0SLQvVahQgY8++ohly5ZRokQJzp49q+OHKugV5zJ+l8vF6dOneeedd2yMSIFNNUUBF7DQqpC0G3c90Ya5V2SMmWqMaW2MaX3VVVddacxBp3Tp0lx11VVkZ2czYMAAfv/73xdYNV2pQFfU89A9tWvXjgYNGmg1Ix/wJqH/G7hWROpaBzqHAItzLfM90A1ARK4GGuCuQ6o8REZG0qxZM6ZMmcKwYcOKdPm0UoGkOIVecqoZrVu3jvT0dBujUnbVFH0G6CAiW4GVwJ+NMYd9FXSwEhGee+45/v73v/P222/Tr18/Tp8+7XRYShVZTExMsV6flJQUNtWM/ElnW3TItGnTSElJoW/fvixatMjpcJQqkkOHDlG9evViraN3795s2bKF/fv3e10wQ+lsiwFp9OjRxMXF6SxxKiilp6cXO6G7XC4GDhzIxx9/TM+ePW2KLLzppf8OGjhw4MWDS//4xz/45ptvHI5IKe/Uq1ev2Ovo06cPlSpVYvr06TZEpEATekDIzMzkhRde4IYbbmDNmjVOh6NUoeyoPlSyZElGjBjBe++9x08//WRDVEoTegCoVKkSGzdupGrVqtx88828//77ToekVIGOHTtmy3rGjBlDdna2Hhy1iSb0ABEfH8/69etp3rw5AwYM4M0333Q6JKXyVZzz0D01bNiQLl26MHXqVC5cCPuLy4tNE3oAqVixIp988gldu3bVo/4qoBXnPPTcUlJS2LNnD6tWrbJtneFKE3qAiYmJ4aOPPuLOO+8EYMuWLbrnogJOXFycbeu6/fbbqVSpklYzsoEm9ACUU0R3165dtG3blqSkJL2qVAWUSpUq2bau0qVL68FRm2hCD2AJCQk89dRTvPXWW9xyyy0cP37c6ZCUArC9GldKSgrZ2dk6v0sxaUIPYCLCI488wuzZs1m9ejXdunXj8GGdUUE5LyEhwdb1NWjQgC5dujB9+nSdjbQYNKEHgeHDh/Pee++xbds2ZsyY4XQ4SnHoUO4JV4vP5XKxe/du1q1bZ/u6w4Um9CDRu3dv/vOf//Dwww8D6PS7ylEnT560fZ0DBw4kJiZGh12Kwa4i0Q+LyBbrtk1EzotIRfvDDW+NGjVCRNi7dy9NmzbVPRnlGLvOQ/dUtmxZBg8ezPz5833yhREObCkSbYz5hzGmuTGmOfAIsNoYc8QH8SrL+fPnuemmm1iwYIHToagwZOd56J5GjhzJqVOn9HN9hewqEu1pKDDXjuBU3urWrcuGDRto1aoVgwcPZtKkSU6HpMKMnacteurQoQMJCQk67HKF7CoSDYCIlAF6Au/m0x52NUV9pVKlSnzyySf06tWLe+65h7lz9TtU+U9sbKxP1isiJCcns2bNGnbv3u2TPkKZXUWic/QB1uc33BKuNUV9JTo6moULF/L000/Tp08fp8NRYWTfvn0+W3dSUhIRERE6YdcVsKtIdI4h6HCLX0VFRfH4448TExPDyZMnef755/UMGOVzjRo18tm6a9SoQY8ePUhNTSU7O9tn/YQiu4pEIyLlgS6Azv3qkAULFjBu3DiGDh3K2bNnnQ5HhTBf7qGD+8rRgwcPsnTpUp/2E2rsKhIN0B9YYYw55ZtQVWGSk5OZOHEi77zzDn379tVTv5TPnDlzxqfr79WrF9WqVWPq1Kk+7SfUaJHoEDRjxgzGjBlDy5Yt+fDDD7n66qudDkmFmBMnTvjswGiOxx9/nOeee469e/dSu3Ztn/YVTAoqEq1XioagkSNH8v7773PkyBFOnz7tdDgqBPnqPHRPo0ePxhjDtGnTfN5XqNCEHqJ69+7Nzp07qVu3LhcuXODbb791OiQVQqpUqeLzPurUqUPPnj2ZPn26Hhz1kib0EBYVFQXA888/T4sWLfjkk08cjkiFipIlS/qln5SUFA4dOsSSJUv80l+w04QeBpKTk7nmmmvo1asXixYtcjocFQIyMjL80k/v3r314GgRaEIPA9WqVSMtLY2WLVsycOBA3njjDadDUkGuSZMmfumnRIkSuFwuli9fzsGDB/3SZzDThB4mKlasyMcff8zNN9/Mvffey/79+50OSQWx9PR0v/Xlcrm4cOECb775pt/6DFaa0MNITEwMixcvZvXq1dSpUwdAq8OoK+LPq5Hr169Pp06dmDlzpn5eC6EJPcxERUXRrl07AObMmcNtt92mpzaqImvatKlf+3O5XKSnp7Nx40a/9htsNKGHsTNnzrBkyRJ69uzJr7/+6nQ4Kohs2bLFr/0NGjSIsmXLagnGQmhCD2Njxoxh7ty5bNq0iRtvvJGff/7Z6ZBUkKhWrZpf+4uJiWHQoEG8/fbbnDqls4vkRxN6mLvjjjtYvHgxO3fupHPnzpw4ccLpkJTKk8vl4uTJk7z7bp7lFhQ21RS1lkm0aopuF5HV9oapfKlnz56sWLGCpKQkn8/PoULDDz/84Pc+O3XqRP369bWaUQFsqSkqInHA68BtxpjrgEH2h6p8qWPHjowfPx5wz9Phj7k6VPBq3ry53/vMqWaUlpbGnj17/N5/MLCrpuidwEJjzPcAxhgdjA1SxhjuvvtuunTpwrJly5wORwWorVu3OtJvUlISIsKsWbMc6T/Q2VVTNAGoICJpIrJZRJLsClD5l4iwePFiEhIS6NOnD9OnT3c6JBWAIiMjHem3Vq1adO/endTUVC5cuOBIDIHMrpqiJYBWQC+gB/C4iCRctiItEh0UqlWrxurVq7npppsYPXo0zz77rF7QoS6RkHDZf2+/GTlyJN9//z2rVq1yLIZAZVdN0QxguTHmlDHmMLAGuD73irRIdPCIjY3lgw8+YPjw4Xz22We6N6QusW3bNsf67tu3L3FxcXpwNA921RR9H+gkIiVEpAzwO9zl6lQQi4qKIjU1lQULFhAZGcmRI0d0XmoFQM2aNR3ru3Tp0tx5550sXLiQY8eOORZHILKlpqgxZgewHPga+ByYZoxx7itc2SYiIoJSpUpx9uxZunXrxoABA3SqAMVvv/3maP8ul4usrCzefvttR+MINFpTVHnttdde4/7776dt27Z88MEH6LBZ+EpLSyMxMdGx/o0xNGvWjDJlyvDZZ585FocTtKaossW9997Lu+++y1dffUWHDh347rvvnA5JOaRVq1aO9i8ijB49ms8//5yvv/7a0VgCiSZ0VST9+/dn1apVHD16lNGjRzsdjnJIIFx4Nnz4cEqVKsWUKVOcDiVgaEJXRda+fXvWrl3L7NmznQ5FOSQ6OtrpEKhYsSKDBw9mzpw5OmGXRRO6uiKNGjWiVq1anD9/nlGjRrFixQqnQ1J+FB8f73QIgLuI9PHjx/XgqEUTuiqW48ePs3nzZnr37s2CBQucDkf5yY4dgXFW8g033EDjxo112MWiCV0VS4UKFUhLS6Nt27bccccdOlVAmAiUPXQRISUlhc8//9zvRTcCkSZ0VWxxcXF89NFH3HzzzYwePZpXXnnF6ZCUjwXSvPnDhw+ndOnSTJ061elQHKcJXdmibNmyvP/++yQlJdGiRQunw1E+lpmZ6XQIF1WsWJFBgwYxZ86csL/oTRO6sk3JkiWZNWsWnTp1AuDdd9/lzJkzDkelfMHp89BzGzVqFCdOnGDhwoVOh+IoTejKJ3bt2sXgwYPp3r27zrcRggLhPHRPnTt35pprrgn7Cbs0oSufaNCgAfPmzePzzz/nxhtv5KeffnI6JGWjmJgYp0O4RE41o1WrVrFv3z6nw3GMJnTlM4MGDeKDDz4gPT2dTp06sX//fqdDUjapXr260yFcZsSIEYgIqampTofiGFuKRFsFon+1ikRvEZG/2B+qCkY9evTg448/5pdffmH1aq0dHirS09OdDuEytWvXplu3bmFdzciWItGWtcaY5tbtaZvjVEGsQ4cOpKenk5Tkrkx49OhRhyNSxVWvXj2nQ8iTy+Vi//79pKWlOR2KI+wqEq1UgXKm2t28eTN169blzTffdDgiVRyBdNqip/79+1O+fPmwPThqV5FogPYi8pWILBOR6/JakdYUVfXr16dly5YkJSXpBUhBLFDPXIqOjmbo0KG8++67ARujL9lVJPo/QB1jzPXAq8B7ea1Ia4qq8uXLs2zZMvr3788DDzzAU089pQWog1CgnYfuacyYMZw5c4Y5c+Y4HYrf2VIk2hhz3Bhz0rq/FIgSkcq2RalCSqlSpZg/fz7Jyck8+eSTzJ8/3+mQVBEF2nnonlq2bEmrVq2YOnVq2O0s2FIkWkSqiohY99ta6w3MQTYVEEqUKMH06dOZNWsWAwcOdDocVURxcXFOh1CgsWPHsnXrVjZt2uR0KH5lS5FoYCCwTUS+Al4Bhphw+2pURRYREUFSUhKRkZEcPHiQP/zhD2RlZTkdlvJCpUqVnA6hQEOGDCEmJibsptX16jx0Y8xSY0yCMaaeMeav1nOTjTGTrfuvGWOuM8Zcb4xpZ4zZ4MugVehJS0vj9ddf55ZbbgnLg1nBJtDrycbGxjJs2DDefvvtsDpNVq8UVQFh2LBhzJkzh/Xr19OpUycOHDhQ+IuUYxISEpwOoVApKSlkZWWF1cFRTegqYAwbNoxly5bx/fff0759e3bu3Ol0SCofhw4dKnwhh7Vs2ZLWrVszZcqUsDk4qgldBZRu3bqxdu1amjRpQtWqVZ0OR+Xj5MmTTofglTFjxrB9+3b+/e9/Ox2KX2hCVwGnWbNmLF++nLi4OLKystiwQQ/JBJpAPg/d05AhQ4iOjg6bK0c1oauA9sQTT5CYmKhV3QNMIJ+H7qlcuXIMGDCAuXPnhkWxFU3oKqCNHz+edu3aMXToUF5++WWnw1GWQD9t0ZPL5eLXX39l0aJFTofic5rQVUArX748H330Ef369ePBBx/kj3/8Y9hOjRpIYmNjnQ7Ba4mJicTHx4fFsIsmdBXwoqOjeeedd7jvvvtITU3l4MGDTocU9oKpKlBERATJycmsXLky5IusaEJXQSEyMpKXX36Zr7/+mlq1amGM4ezZs06HFbYaNWrkdAhFMmLECIwxzJo1y+lQfEoTugoaIkKNGu6Zm5955hluvPHGsLoKMJAE0x46QHx8PF27dg35akaa0FVQatKkCZs3b6ZLly46BOOAYDxjxOVysXfvXtasWeN0KD5jS01Rj+XaiMh5EdHp85RP3X777SxZsoS9e/fSvn17tm/f7nRIYSVYzkP3dPvtt1OuXLmQPjhqW01Ra7nncc/KqJTP3XTTTaxZs4bs7GwSExM5fvy40yGFjWA5D91TmTJlGDJkCAsWLODEiRNOh+MTdtYUvQ94F/jZxviUKlCLFi3YuHEjkyZNoly5ck6HEzaqVKnidAhXxOVycfr06ZAtqmJLTVERqQH0BybbF5pS3qlTp87FIhnvv/9+SP+kDhQlS5Z0OoQr8rvf/Y6GDRuG7GfErpqiLwF/NsacL3BFWiRa+ZAxhhkzZjBy5EheeOEFp8MJaRkZGU6HcEVEBJfLxfr160lPT3c6HNvZUlMUaA3ME5F9uKsXvS4i/XKvSItEK18SEebPn8/AgQP54x//yOOPPx4206b6W5MmTZwO4YoNHz6cyMhIUlNTnQ7FdrbUFDXG1DXGxBtj4oEFwD3GmPfsDlapwpQqVYp58+YxatQonn32We67776QPu/YKcG8d1utWjVuueUWZs2axfnzBQ4qBB27aooqFTAiIyN54403eOihh4iMjMSqX65sFOyJcOTIkRw6dIhly5Y5HYqtxKmfpK1btzZffPGFI32r8JDz2RYR0tPTqVatWlBNKhXIjh07RlxcnNNhXLFz585Ru3Zt2rRpw+LFiwt/QQARkc3GmNZ5temVoipkiQgiwtmzZ+nRowedO3fmhx9+cDqskLBlyxanQyiWqKgoRo0axZIlS0Kqfq0mdBXySpUqxaRJk/j2229p166d1iq1QbVq1ZwOodhGjx598cyoUKEJXYWFnj17smbNGrKysujUqRP/+c9/nA5JOSw+Pp4ePXowbdo0srOznQ7HFprQVdho2bIla9eupUyZMjz99NNOhxPUQmXoKiUlhYyMDJYvX+50KLbQg6Iq7Bw8eJBy5coRGxvL+fPniYyMdDqkoBPsB0VznDt3jjp16tCqVSs++OADp8Pxih4UVcpDjRo1iI2N5fTp0yQmJvL66687HVLQ2bp1q9Mh2CIqKoqRI0eydOnSkDg4qgldhS1jDHFxcfzhD39g3LhxegFSEYTSr5pRo0Zx4cIFZs+e7XQoxaYJXYWtsmXLsmjRIsaOHcvzzz/PqFGjQubgmK8lJCQ4HYJt6tatS2JiIqmpqUE/VYQmdBXWSpQowaRJk3jyySdJTU3lf/7nf5wOKShs27bN6RBs5XK52L17N+vWrXM6lGLRhK7CnojwxBNPMHnyZB544AGnwwkKNWvWdDoEWw0YMIDY2Nign1ZXE7pSlrFjx1K/fn2MMfz973/nyJEjTocUsH777TenQ7BV2bJlGTx4MPPnz+fkyZNOh3PFNKErlcv27dt5/PHH6dy5M4cO5Z4pWgH8/HPoFSYbOXIkp06d4p133nE6lCtmS5FoEekrIl+LyBargEVH+0NVyj+aNGnCsmXL2L9/Px07dmT37t1OhxRwgrFIdGHat29PgwYNgnrYxa4i0SuB640xzYGRwDSb41TKr7p27cqqVas4fvw4HTp04PPPP3c6pIASjEWiCyMiJCcns3btWr799lunw7kithSJNsacNP8936csl5eoUyrotGnThg0bNlCxYkWOHTvmdDgBJTo62ukQfGLEiBEX59MPRrYUiQYQkf4ishNYgnsv/TJaU1QFm4SEBLZu3crNN98MwK5duxyOKDDEx8c7HYJPVKtWjdtuu42ZM2dy9uxZp8MpMruKRGOMWWSMaQj0A57Ja0VaU1QFo6ioKABWrVpF48aNmThxosMROW/Hjh1Oh+AzY8eO5fDhw7z33ntOh1JkdhWJvsgYswaoJyKVixmbUgGlY8eODBo0iIcffpjx48cH/VWFxRGqe+gA3bt3Jz4+nqlTpzodSpHZUiRaROqLVbhRRFoCJYFMu4NVykklS5bkrbfeIiUlhQkTJjB27NiwnSrgxIkTTofgMxEREYwZM4ZVq1YF3cFRu4pEDwC2icgW3GfE3GHCefdFhazIyEgmT57Mo48+yhtvvBE0U67aLTMztPfXXC4XJUqUCLq9dJ0PXakrtGHDBjp06AC4Z260fqSGhRMnToR8we0BAwawZs0aMjIyKFWqlNPhXKTzoSvlAznJ/Msvv6Rt27bs37/f4Yj8JxTPQ88tJSWFw4cP8/777zsditc0oStVTFlZWezevZsbbrghpM/+8BQTE+N0CD530003UbNmzaC6clQTulLF1L59e1avXk12djadOnUiHIYSq1ev7nQIPhcZGcmIESNYsWIFBw8edDocr2hCV8oGzZo1Y926dcTGxtK1a1e+/vprp0PyqfT0dKdD8Ivk5OSgqmakCV0pm9SvX59169aRnJxMw4YNnQ7Hp+rVq+d0CH5Rv359OnfuzMyZM4PiugNN6ErZqEaNGrzyyiuULFmSX375hQULFjgdkk+E+mmLnlwuF99++y3r1693OpRCaUJXykeeffZZBg0axHPPPRcUe3dFEU6TlQ0cOJCyZcsGxcFRTehK+cg//vEPhg0bxqOPPso999wTUleVhuJ86PmJiYm5WM3o1KlTTodTIE3oSvlIyZIlmT17NuPGjWPy5MkMGDCArKwsp8OyRTich+7J5XJx8uTJgB9C04SulA9FREQwYcIEXn31VXbt2sXx48edDskWcXFxTofgVx07dqR+/foBP+yiCV0pP7j33nvZsmULVapU4dy5c0FfgLpSpUpOh+BXIoLL5WL16tXs2bPH6XDyZVdN0WFWTdGvRWSDiFxvf6hKBbfSpUsD8Pvf/54OHTqwd+9ehyO6ct99953TIfhdUlISERERpKamOh1KvuyqKboX6GKMaYa7uEVwTVGmlB+NGDGCn376ifbt2wftWHRCQoLTIfhdzZo16d69O6mpqZw/f97pcPJkV03RDcaYo9bDTbiLYCil8tCpUyfWr19PqVKl6NKlC8uXL3c6pCI7dCjfGjchzeVyceDAAVatWuV0KHmyraaoh1HAsuIEpVSoa9y4MRs3buTaa69l2LBhQXew9OTJk06H4Ii+ffsSFxcXsAdHS3ixjFc1RQFE5EbcCb1jPu0pQApA7dq1vQxRqdBUvXp10tLSSE9Pp1y5ck6HUyThdB66p9KlS3PnnXcyY8YMjh07FnBn+9hWU1REmgHTgL7GmDyvC9Yi0Updqnz58rRp0waAF198kQkTJgTFVaXBOvZvh5EjR5KVlcX//d//OR3KZeyqKVobWAgMN8aExzRsStnIGMPmzZsZP3489957b8AedMsRbqctemrZsiUtWrRgypQpAffla1dN0b8AlYDXRWSLiIT+hNBK2UhEmD17Ng8//DCvv/46AwYM4PTp006Hla9QLz9XEBFh7NixfP3113z++edOh3MJrSmqVIB59dVXeeCBBy4WzihRwptDXf6VlpZGYmKi02E45vjx41SvXp077riD6dOn+7XvgmqKBt4nRakwd99991GjRg1+/PHHgEzmAI0aNXI6BEeVK1eOO++8k7feeosXXniB8uXLOx0SoJf+KxWQbr/9du655x6AgLzcfN++fU6H4LixY8dy+vRp3nrrLadDuUgTulIB7OzZsyQlJdGxY0e2bdvmdDgXnTlzxukQHNeqVStatmwZUAdHNaErFcBKlSrF0qVLAejSpUvAHIQL1/PQc8s5OPrZZ585HQqgCV2pgHfdddexbt064uLi6NatW0Bcdh7O56F7Gjp0KGXLlmXGjBlOhwJoQlcqKFxzzTWsW7eO+Ph45s+f73Q4VKlSxekQAkJsbCwDBw5k3rx5AXGaqSZ0pYJEtWrVWLt2La+++ioAR44ccWzstmTJko70G4hcLhcnTpxg4cKFToeiCV2pYBIXF0dUVBRHjhyhdevWPPjgg1y4cMHvcWRkZPi9z0DVuXNn6tWrFxDDLprQlQpCcXFx9OvXj1deeYWkpCTOnTvn1/6bNGni1/4CmYiQnJzMp59+6njREk3oSgWhiIgIXnjhBSZMmMBbb73F7bff7tdTCdPTdcomTyNGjEBEmDVrlqNxaEJXKoiNGzeOSZMmsWTJEsaPH++3fgN98jB/q1WrFjfddBOpqamODIHl0ISuVJC7++67WbRoEY899pjf+mzatKnf+goWLpeL/fv38+mnnzoWg11FohuKyEYROSsiD9kfplKqIH379qVSpUr89ttv3HXXXXz11Vc+7W/Lli0+XX8w6tevHxUqVOCNN95wLAa7ikQfAe4HJtoeoVLKawcPHiQtLY3OnTuzZs0an/VTrVo1n607WEVHR5OUlMTChQv55ZdfHInBriLRPxtj/g3491C7UuoSdevWZdOmTdSoUYMePXqwZMkSp0MKKykpKZw7d47U1FRH+vdFkeh8iUiKiHwhIl849Q2mVKirWbMma9asoUmTJvTr148FCxbY3scPP/xg+zpDQePGjenYsSNTp0515KIvbxK610WiC6M1RZXyj8qVK7Ny5Ur69OlDgwYNbF9/8+bNbV9nqBg7diy7d+925OCobUWilVKBpVy5cixcuJCmTZtijOGTTz6xba9x69attqwnFA0YMIAKFSowZcoUv/dtS5FopVRgW7x4Md27d+fhhx+25TzpyMhIG6IKTdHR0YwYMYJFixbx888/+7VvW4pEi0hVEckA/hd4TEQyRKScLwNXSnmvT58+3Hvvvfzzn/8kKSmJ3377rVjrS0hIsCmy0JRzcNTfV45qkWilwoQxhr/97W+MHz+erl27snDhwiuuhRnuRaK90aFDB44dO8b27dsRyetQ5JUpqEi0XimqVJgQER555BFmz57N+vXr2bRp0xWvq2bNmjZGFppGjhzJjh07/FplShO6UmFm+PDh7Nmzhx49egCQlZVV5HUUd8gmHAwePJjo6Ghmzpzptz41oSsVhqpXrw7A0qVLSUhIKPJZK/4+2BeMypUrx8CBA5k7d67fqhlpQlcqjMXHx3P+/Hm6dOnCunXrvH6dFon2jsvl4vjx4yxatMgv/WlCVyqMNW7cmHXr1lG5cmVuuukm3n33Xa9ep0WivdOlSxfi4+P9NuyiCV2pMFe3bl02bNhAy5YtGTRokFcH8aKjo/0QWfCLiIggOTmZVatWsX//ft/35/MelFIBL2eqgMmTJ9OmTZtCl4+Pj/d9UCEiOTkZwC976ZrQlVKAe687JSUFEWHnzp38+c9/zrcy0Y4dO/wcXfCqU6cO3bt3Z/r06T6v9KQJXSl1mcWLF/P3v/+du+66K89TFHUPvWhSUlLIyMhg2bJlPu1HE7pS6jJ/+tOfeP7555k3bx79+vW77LS7EydOOBRZcLrtttu4+uqrmTp1qk/70YSulMrTn/70J6ZOncry5cvp0aMHv/7668W2zMxMByMLPlFRUYwcOZIlS5Zw4MCBwl9wheyqKSoi8orV/rWItLQ/VKWUv40ZM4Z58+ZRqlQpoqKiLj6v56EX3ZgxYzDGMH36dJ/1UejkXFZN0XSgO+650f8NDDXGfOOxzK3AfcCtwO+Al40xvytovTo5l1LBwxiDiJCZmUl6ejpnz57VybmuQM+ePdm+fTt79+6lRIkSV7SOgibn8maNF2uKWivLqSn6jccyfYHZxv3tsElE4kSkmjFG61QpFQJyZgt85JFHmDlzJg888ADXXnstAGXKlKFChQqAu0h1bmXLliUuLg5jDIcOXV4bJyYmhvLly3P+/Hl+/PHHy9rLlStHbGws2dnZ/PTTT5e1ly9fnpiYGM6dO5fnlAQVKlSgTJkynD17lsOHD1/WXrFiRaKjo8nKyspzKKlSpUqULl2aM2fOcOTIkcvar7rqKkqWLMmpU6c4duzYZe1VqlQhKiqKkydPMmDAAD766COWLVtGnz59Llu22IwxBd6AgcA0j8fDgddyLfMh0NHj8UqgdUHrbdWqlVFKBZejR4+aPn36GNxlKA1gXC7XxfYSJUpc0gaY+++/3xhjzJkzZy5rA8z48eONMcb8/PPPebZPmDDBGGPMd999l2f7a6+9ZowxZsuWLXm2z5o1yxhjzLp16/JsX7hwoTHGmGXLluXZvmLFCmOMMfPnz8+zfePGjcYYY6ZPn55n+7Zt24wxxrz00ksGMFFRURe3+UoAX5h88qo3e+je1BT1qu6oiKQAKQC1a9f2omulVCCJi4vjvffe469//StVq1YFLi12MWnSpMvK3F133XUAlChRIs+zPHLqk8bExOTZnnOhU+XKlfNs79ChA+Ce0jev9nbt2gFQr169PNtbtGhxMc682hs1agS4jxvk1V63bl0AbrjhhjzbcyZC69atG1OnTqVs2bLceeedly1nB2/G0NsDTxpjeliPHwEwxkzwWGYKkGaMmWs93gUkmgKGXHQMXangdeDAAWrVqlX4gsp2xS1w4U1N0cVAknW2Szvg14KSuVIquOlpi4Gp0CEXY0y2iOTUFI0EZhirpqjVPhlYivsMl93AacDlu5CVUk7L6+Cfcp7WFFVKFdmJEyeIjY11OoywpDVFlVK20vnQA5Nje+gi8gtwpRMEVwYuP6HUP5zqW7c5PPoOt36d7DtYt7mOMeaqvBocS+jFISJf5PeTI1T71m0Oj77DrV8n+w7FbdYhF6WUChGa0JVSKkQEa0L37aTCgdm3bnN49B1u/TrZd8htc1COoSullLpcsO6hK6WUykUTulJKhYiATuhOVUryot+GIrJRRM6KyEN29FmEvodZ2/q1iGwQkev91G9fq88tIvKFiHS0o19v+vZYro2InBeRgf7oV0QSReRXa5u3iMhf7OjXm749+t8iIttFZLU/+hWRhz22d5v1flf0U9/lReQDEfnK2mZbphDxot8KIrLI+nx/LiJNbOp3hoj8LCLb8mm3P3/lN6+u0zfc88Z8B1wDlAS+AhrnWuZWYBnu6XvbAZ/5qd8qQBvgr8BDft7mDkAF6/4tftzmGP57zKUZsNNf2+yx3Crc8wYN9NM2JwIfOvTZjsNdRKZ2zmfOX++1x/J9gFV+3ObxwPPW/auAI0BJP/T7D+AJ635DYKVN29wZaAlsy6fd9vwVyHvoFyslGWN+A3IqJXm6WCnJGLMJiBORar7u1xjzszHm38C5YvZ1JX1vMMYctR5uAmr6qd+TxvoUAmXJY757X/VtuQ94F7i8JI1v+/UFb/q+E1hojPke3J85P/XraSgw14Z+ve3bALEiIrh3II4A2X7otzHuojwYY3YC8SJydTH7xRizBvc25Mf2/BXICb0G4FkeO8N6rqjL+KJfXylq36Nwf8P7pV8R6S8iO4ElwEgb+vWqbxGpAfQHJtvUp1f9WtpbQwDLROQ6P/adAFQQkTQR2SwiSX7qFwARKQP0xP0lagdv+n4NaAQcArYCDxhjLvih36+A2wFEpC1QB3t2lOyIrUgCOaHbVinJB/36itd9i8iNuBP6n/3VrzFmkTGmIdAPeMaGfr3t+yXgz8aY8zb16W2//8E9b8b1wKvAe37suwTQCugF9AAeF5GEy15lf785+gDrjTEF7WHa3XcPYAtQHWgOvCYi5fzQ799wf3luwf1L8EuK/8vAG7bnmisrO+0fGYBnSZSauL+5i7qML/r1Fa/6FpFmwDTgFmOMHZUGirTNxpg1IlJPRCobY4o7uZE3fbcG5rl/iVMZuFVEso0x7/myX2PMcY/7S0XkdT9ucwZw2BhzCjglImuA64F0H/ebYwj2Dbd427cL+Js1tLdbRPbiHtP+3Jf9Wn9nF7gPVAJ7rZuv2Z9r7Bj898UN95fNHqAu/z2YcV2uZXpx6UGFz/3Rr8eyT2LvQVFvtrk27kIiHfzcb33+e1C0JXAw57G/3m9r+VTsOSjqzTZX9djmtsD3/tpm3EMPK61lywDbgCb+eK+B8rjHfsv6+TM2CXe5S4Crrc9YZT/0G4d18BUYg3tc267tjif/g6L25y+7AvfFDfdR4HTcR6kftZ67G7jbui/Av6z2rUBrP/VbFfe363HgmHW/nJ/6ngYcxf3TdAsFVAC3ud8/A9utPjcCHf31d861bCo2JHQvt/lea5u/wn0A2s4v0UK3GXgY95ku24AH/dhvMjDPrm0twvtdHVhh/V/eBtzlp37bA98CO4GFWGeR2dDvXOAH3CdPZOAeIvVp/tJL/5VSKkQE8kFRpZRSRaAJXSmlQoQmdKWUChGa0JVSKkRoQldKqRChCV0FBRGp5DEL4I8ictC6f0xEvvFBf09KEWfSFJGT+TyfatcMkUoVRBO6CgrGmExjTHNjTHPcc7q8aN1vDhQ634eIBPJV0UrZQhO6CgWRIvKGNYf2ChGJBrAmtnrOmkv8ARFpJSKrrcmuPsqZ2U5E7heRb6w5qed5rLextY49InJ/zpMi8r/WXOHbROTB3MFY81y/Zq1zCe7plnPa/ubR10RfvSEqPOleiwoF1wJDjTFjRGQ+MACYY7XFGWO6iEgUsBroa4z5RUTuwD2f/UhgHFDXGHNWROI81tsQuBGIBXaJyCTcc8G7gN/hvtLvMxFZbYz50uN1/YEGQFPcl7B/A8ywCkX0BxoaY0yuvpQqNk3oKhTsNcZsse5vxj1/Ro63rX8bAE2Aj61JviJxX5YN8DXwloi8x6UzKi4xxpwFzorIz7iTc0dgkXFPmoWILAQ64Z6hL0dnYK5xzw55SERWWc8fB7KAadae+4dXvslKXU6HXFQoOOtx/zyX7qicsv4VYHvOOLwxpqkx5marrRfuOTVaAZs9xtvzWm9eU57mJa/ph7NxT/L1Lu4piJd7uS6lvKIJXYWLXcBVItIeQESiROQ6EYkAahljPgX+hHvmvZgC1rMG6CciZUSkLO4hlLV5LDNERCKtcfobrT5jgPLGmKXAg7gP6CplGx1yUWHBGPObdergKyJSHvdn/yXcs/DNsZ4T3GfPHLOGZfJaz39EJJX/ztE9Ldf4OcAioCvuGfTScY/dg3ss/n0RKW319T82bZ5SADrbolJKhQodclFKqRChCV0ppUKEJnSllAoRmtCVUipEaEJXSqkQoQldKaVChCZ0pZQKEf8PmCzFBswleAkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(6, 4))\n",
    "\n",
    "plt.plot(df_ideal.threshold, df_ideal.tpr, color='black', linestyle='solid', label='TPR')\n",
    "plt.plot(df_ideal.threshold, df_ideal.fpr, color='black', linestyle='dashed', label='FPR')\n",
    "plt.legend()\n",
    "\n",
    "plt.xticks(np.linspace(0, 1, 11))\n",
    "plt.yticks(np.linspace(0, 1, 11))\n",
    "\n",
    "plt.vlines(1 - y_val.mean(), -1, 2, linewidth=0.5, linestyle='dashed', color='grey')\n",
    "plt.ylim(-0.03, 1.03)\n",
    "\n",
    "plt.xlabel('Thresholds')\n",
    "plt.title('TPR and FPR for the ideal model')\n",
    "\n",
    "# plt.savefig('04_fpr_tpr_plot_ideal.svg')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABCPElEQVR4nO3dd3hVZbr38e8dOoQiTYJINfQqTVCaHEHaiB1E6jBiZThHR8cu6jtzHD06Y2VQHFApIhhEpTkCZmgCgUBASCgSCKAJEJCEJCTZ9/vH3sSAIdmB7Kxd7s915cpea6+91m8FcudZ7XlEVTHGGOOdMKcDGGNMILGiaYwxxWBF0xhjisGKpjHGFIMVTWOMKQYrmsYYUwxWNI0xphisaBqfE5EDIpIhImki8pOIzBSR8AuW6SkiK0XktIicEpEvRaT1BctUE5G/i8hBz7r2eqZrl+4emVBmRdOUlmGqGg50BDoBT557Q0R6ACuAL4D6QBNgG7BWRJp6likPfAu0AW4GqgE9geNAN1+FFpGyvlq3CUxWNE2pUtWfgOW4i+c5fwM+UtV/qOppVT2hqs8AG4AXPMuMARoCt6rqD6rqUtVkVX1JVZcUtC0RaSMi34jICRH5WUSe8syfKSIv51uur4gk5Zs+ICJPiMh2IF1EnhGRBRes+x8i8qbndXURmSEiR0XksIi8LCJlLu8nZfyVFU1TqkSkATAI2OuZroy7xfhZAYvPB27yvP4vYJmqpnm5narAv4FluFuv1+BuqXprJDAEqAF8DAwWkWqedZcB7gLmeJadBeR4ttEJGABMLMa2TACxomlKyyIROQ0cApKB5z3za+L+f3i0gM8cBc6dr6x1kWUuZijwk6r+n6pmelqw3xfj82+q6iFVzVDVRGALMNzz3o3AGVXdICJX4v4jMEVV01U1GXgDGFGMbZkAYkXTlJbhqloV6Au05NdimAq4gIgCPhMBHPO8Pn6RZS7mamDfJSV1O3TB9BzcrU+Ae/i1ldkIKAccFZGTInIS+CdQ9zK2bfyYFU1TqlT1O2Am8JpnOh1YD9xZwOJ38esh9b+BgSJSxctNHQKaXeS9dKByvul6BUW9YPozoK/n9MKt/Fo0DwFZQG1VreH5qqaqbbzMaQKMFU3jhL8DN4lIR8/0n4GxIjJZRKqKyBWeCzU9gKmeZT7GXaAWikhLEQkTkVoi8pSIDC5gG18B9URkiohU8Ky3u+e9WNznKGuKSD1gSlGBVTUFWA38C/hRVXd55h/FfeX//zy3RIWJSDMR6VPMn4kJEFY0TanzFKCPgGc902uAgcBtuM9bJuK+oHKDqu7xLJOF+2LQbuAb4BdgI+7D/N+cq1TV07gvIg0DfgL2AP08b3+M+5amA7gL3qdeRp/jyTDngvljgPLAD7hPNyygeKcSTAAR64TYGGO8Zy1NY4wpBiuaxhhTDFY0jTGmGKxoGmNMMVjRNMaYYgi4Hlxq166tjRs3djqGMSbIxMTEHFPVOkUt57OiKSIf4n7+N1lV2xbwvgD/AAYDZ4BxqrqlqPU2btyYzZs3l3RcY0yIE5FEb5bz5eH5TNz9Hl7MICDS83Uf8J4PsxhjTInwWdFU1WjgRCGL3IK7D0VV1Q1ADRGxpyiMMX7NyXOaV3F+TzJJnnnF6f7rN1wuF/aUkzGhJTs7m/T0dNLT00lLSzvv+4XzT58+Tbdu3Rg+fPglbcvJoikFzCuw2onIfbgP4WnYsOFv3s/NzWX//v0cPHgQEaFcuXIlGtQYc/lUlbNnz5KZmUlGRgaZmZmcOXPmsqYzMjLIyMggJyenWFkmTZoUkEUzCXefh+c0AI4UtKCqTgemA3Tp0uU3hfXbb7+lRYsW3HjjjbivLxljLoeqkpGRQVpaWt7X6dOnz5u+2LyLzU9PTyc3N9frDGXLliU8PJwqVarkfa9evTr169f/zXxvvmdmZrJo0SKGDh1K27a/uTbtfa5L/uTlWww8LCLzgO7AKU83W8VWvnx5GjVqVKLhjAlGqkpycjI//vgj+/fvP+/74cOHzyt43p7mCgsLIzw8/LyvqlWrEhERkVe0qlatWuwiV758+RLd73/+858MHDjwsgom+PaWo7m4e+mu7Rm06nncPVyjqtOAJbhvN9qL+5aj8b7KYkwwOXv2LPv27ePkyZMXXUZVOXbsWIHF8cyZM+ctW69ePZo0aULHjh2pVq3abwrguSJ4sfkVK1b06yO8rKwsypcvz9ixY6lUqdJlr89nRVNVRxbxvgIP+Wr7xgS6EydOsHv37vO+4uPj2bdvX7EOc6tUqULTpk1p1qwZN910E02bNqVJkyY0bdqUxo0bU7ly5aJXEqDOnj3Lxx9/zHXXXXfZLcxzAu6JIGMCmary888/c/z48byvEydO5L1OSUlh79697N69m5SUlLzPlS9fnubNm9O+fXvuuusuWrZsSe3atQvZEtSoUYOmTZtSu3Ztv24J+kpubi6fffYZtWvXpk2bkht9xIqmMT7mcrlYv349CxYsYOHChRw6dOGYbW7ly5enVq1aXHPNNQwfPpyWLVvmfTVq1IgyZWwo9eLYtm0bIsKwYcNK9I+GFU1jfCA3N5c1a9bkFcqjR49SoUIFBg4cyJ/+9CeuvPJKatWqRc2aNfO+V6lSJSRbhCUpMzOTzZs3U69ePTp16kT79u1L/I+NFU1jiiEzM5PFixeTlpZW4PuqyqZNm4iKiiI5OZmKFSsyePBg7rjjDoYMGUK1atVKOXFoyM7OZvXq1WzZsoXIyEhatGiBiFC2bMmXOCuaxnjB5XIxb948nnrqKRITC+/XoXLlygwdOpQ77riDQYMGER4eXkopQ09mZiapqanUq1eP8uXLc//991O9enWfbtOKpjFFWL16NY899hgxMTF06tSJf/7zn7Rq1eqiy9epU6dEbm0xhYuPj+fLL7+kU6dORERE0KdP6YyabEXTmIv44YcfeOKJJ/jqq6+4+uqr+eijjxg1ahRhYdZ3t9PWrFlDTEwMd955Z6k/2GJF05h8zpw5w6pVq1iwYAEfffQR4eHh/O///i+TJ0+21qOfcLlctG3blq5du1KhQoVS374VTRPy9u/fz5IlS1iyZAmrVq0iMzOTKlWq8PDDD/Pss88WeT+kKT3btm1j37593HbbbY5lsKJpQtKPP/7IW2+9xZIlS4iPjwegefPm3H///QwZMoRevXo50ooxF3f06FGWL1/OuHHjHM1hRdOElNzcXN5++22eeuopcnJy6NevHw8++CCDBw/mmmuucTqeuYjMzEzmzJnDkCFDqFu3rqNZrGiaoKeq/PTTT+zYsYPnnnuODRs2MGTIEN577z2uvvrqoldgHJWWlkZ4eDjjx4+nZs2aTsexommCS1JSEuvXryc+Pj6vg4uEhAR++eUXAGrXrs2cOXMYMWKEPX3jx2JiYoiLi+PYsWMAPPTQQ35RMMGKpgkCqsqaNWv4xz/+QVRUFC6XC4Crr76ali1bMmbMGFq0aEGLFi3o1q2bz29+Npevbt269OrVizp16lC1alW/+gNnRdMEtKVLl/LMM8+wZcsWrrjiCv70pz9x11130aJFC6pUqeJ0PFNMOTk5zJs3j7vvvttvh62xomkC0s8//8yUKVOYN28ekZGRTJs2jdGjRwd135ChYPfu3eTm5vptwQQrmiYApKWlsWvXLnbu3ElsbCwbN25ky5YtqCpTp07liSeesNuDgkRMTAydO3d2OkahrGgav6KqfPHFF6xdu5YffviBnTt3ntdBRqVKlejcuTMPP/wwEydOpGXLlg6mNSUpOzubsLAwv/83taJp/EZqaip/+MMfWLhwIeXLl6dly5b07NmTP/zhD7Ru3Zo2bdrQtGlTn3T3ZZylqqgqo0ePdjpKkex/n/EL69evZ+TIkRw+fJhXX32VKVOmWHEMASdOnODnn3/mxx9/5JdffmHEiBFORyqSdddiHOVyuXjllVfo1asXYWFhrF27lscee8wKZpA7NyLm+vXriY2NpVy5cgwcONDhVN6x/5nGMcnJyYwZM4bly5dz55138v7779s9lEEsJyeHY8eOcejQIb777jsefPBBhgwZ4nSsYrOiaRyxatUqRo0axYkTJ5g2bRr33XefX93AbEpOeno6FSpUYM+ePaxcuZI6deowcuTIgL09zIqmKVU5OTm8/PLLvPjiizRv3pxly5bRvn17p2MZH8nNzWXu3Ll07tyZTp06FdrjfaCwomlKzcGDBxk1ahRr1qxh9OjRvPvuuzZ+TpBbvnw54eHhdOzY0ekoJcYuBJlSsXDhQjp06EBsbCwff/xxXq/oJngdPXqUffv2MXz48KA69WJF0/hUeno6kyZN4o477iAyMpKtW7dy7733Oh3L+JCqkpycTEREBJMmTaJixYpORypRVjSNz2zdupUuXbrw/vvv8/jjj7NmzRrr6DfInT17lqioqLzepsqXL+90pBJnRdOUOJfLxRtvvMF1113HL7/8wjfffMMrr7wSlL9A5lcnTpzg/fffR0SYMGFC0I7aaReCTIlKSUlh9OjRLF++nFtuuYUPPvjABiYLERUqVKBv3760bt06qM5hXig4/xQYR5w7HF+9ejXvvfceUVFRVjBDxMaNG3G5XLRp0yaoCyZY0TQlZM6cOVx//fW4XC7WrFnD/fffH/S/PMZt27ZtrFu3zq/7wCxJVjTNZcnJyeGxxx5j1KhRdO3alZiYGLp06eJ0LFNKNm3axLfffss999wTdFfJL8bOaZoi7d69m6SkpLzp9PR0EhISiI+PZ+PGjcTFxfHQQw/xxhtvhExrI9Tl5ORQpkwZqlWrxoQJE6hRo4bTkUqNFU1zUfv27eOZZ55h3rx5Bb5ft25dWrRowaxZsxgzZkwppzNOSU5OZsGCBQwaNIgWLVo4HafUWdE050lOTmb+/PnMmzePtWvXUqlSJZ5++mkGDhyYd46yQoUKREZGhlTrwridOnWKmTNnMmDAABo3bux0HEdY0TR5Dh48SIcOHTh58iTt2rXjL3/5C2PHjqV+/fpORzN+IiEhgbZt2wbVs+TFZUXTAO5H3x588EHOnj3Lpk2b7GKOKVDXrl3Jzc11Ooaj7Oq5AeCzzz7j66+/5qWXXrKCaQp04MABYmNjKVOmjNNRHGVF05CamsrkyZPp3LkzkydPdjqO8VMbNmzA5XI5HcNxVjQNjz32GMeOHeODDz6wsXlMgU6fPk1iYiJt27Z1OorjrGiGuK+//poPP/yQP/3pTyF9ct8U7lzBtE5X7EJQSDt+/DgTJ06kbdu2vPDCC07HMX5KVWnbti2tW7d2OopfsJZmCHvkkUc4duwYH330ERUqVHA6jvFT8+fP5+DBg0Hb1Vtx+fSnICI3i0i8iOwVkT8X8H51EflSRLaJyE4RGe/LPOZXn332GXPnzuX555+nU6dOTscxfiohIYGUlBSuuuoqp6P4DZ8VTREpA7wDDAJaAyNF5ML2/UPAD6raAegL/J+I2EkTH9u7dy8TJ06kW7du/PnPv/lbZgzg7mNg6dKlDBo0KORvM8rPly3NbsBeVd2vqmeBecAtFyyjQFVxP58XDpwAcnyYKeSdOXOG2267jbJlyzJ//ny7Wm4KpKqUL1+ePn360KxZM6fj+BVfFs2rgEP5ppM88/J7G2gFHAHigD+q6m9uBBOR+0Rks4hsTklJ8VXeoKeqTJo0iR07djBnzhwaNWrkdCTjpxYuXEhubq7dUVEAXxbNgnqg1QumBwKxQH2gI/C2iFT7zYdUp6tqF1XtUqdOnZLOGRLOnj3LM888wyeffMLUqVMZOHCg05GMn0pJSeHgwYN2e9FF+LJoJgFX55tugLtFmd944HN12wv8CLT0YaaQtGHDBjp37sxf/vIXxowZw9NPP+10JOPHtm3bRrt27exq+UX48qeyCYgUkSaeizsjgMUXLHMQ6A8gIlcCLYD9PswUMlwuF0uWLGHAgAH06NGD1NRUvvjiC2bNmmW/DKZQKSkptG/f3ukYfstnVwFUNUdEHgaWA2WAD1V1p4jc73l/GvASMFNE4nAfzj+hqsd8lSlUzJ8/n2effZaEhAQiIiJ46aWXmDx5MtWq/ebMhzEAZGRksH//fsqXL8/IkSOdjuPXfHrpVFWXAEsumDct3+sjwABfZgg1W7Zs4Z577qFt27bMnj2bO+64w85NmYs6e/Ysn376KUlJSTRs2JDOnTs7Hcnv2f0mQSQ7O5vf//731KlTh1WrVnHFFVc4Hcn4uXLlynHDDTfQoEEDG9/JS1Y0g8hrr71GbGwsUVFRVjBNkbZv306FChVCcpyfy2FXBILEkSNHePnll7n11lsZPny403GMnzt27BjLli2zP66XwIpmkHjqqafIycnh1VdfdTqK8XNZWVl89tln9OvXj7p16zodJ+BY0QwCmzdvZtasWUyZMsUeeTNFysnJoUOHDjasySWyohng4uPjGTVqFHXr1rWb1k2RNm3ahIjQs2fPvCGZTfFY0Qxghw8fpnv37qSmprJgwQK7D9MUKiYmhu+//96K5WWyq+cB7I033iAtLY0ffviB5s2bOx3H+LEjR47w7bff8vvf/55KlSo5HSegWUszQKWmpvLPf/6TESNGWME0RUpNTWXYsGHUqlXL6SgBz1qaAeqtt94iLS2Nxx9/3Okoxo+pKvv27aNNmzZORwka1tIMMPPmzaNFixY8//zzDBkyxDpWMBelqqxevZro6Ggbr7wEWdEMIEuXLuXee+8lPDyct99+m9mzZzsdyfip7OxsvvjiC3bt2sWdd95pPVuVIDs8DxBbt27lzjvvpH379qxevZqqVas6Hcn4MRGhRo0aDB482DpsKWH25ycAnDx5kmHDhlGzZk2++uorK5jmonJycpg/fz5nz56lb9++VjB9wFqaAeCJJ57g6NGjfP/999SvX9/pOMaPrVixAhGx24p8yFqafu6bb75h+vTp/Pd//7c99mYKtXfvXuLj4xk2bJjdwO5DVjT92O7du7nzzjtp27YtU6dOdTqO8XPlypXj1ltvpWLFik5HCWp2eO6nsrKyuOOOO6hQoQJffvklVapUcTqS8WPx8fFcc801lClTxukoQc9amn5q6tSp7Ny5k5kzZ9K4cWOn4xg/tmXLFpYsWUJOTo7TUUKCtTT90KZNm3jllVcYP348gwYNcjqO8VOqytq1a9m8eTNjxoyhQoUKTkcKCVY0/czWrVu56667iIiI4PXXX3c6jvFzLpeLCRMmWA9XpcgOz/2EqjJt2jR69OhBdnY2n3/+OTVq1HA6lvFTP//8M3v37qV3795WMEuZFU0/kJmZybhx43jggQfo168fW7dupVu3bk7HMn5s69atHDlyxOkYIcmKph945JFH+Oijj3jhhRf4+uuvqVOnjtORjB9TVXbv3k3Lli2djhKS7Jymw+bPn88HH3zAk08+yfPPP+90HBMATp48iYjYoGgOsZamg06dOsUDDzxA9+7d7eZ145Xc3FyuuOIKRo0aZU/9OMSKpoP+/ve/c+LECd577z3KlSvndBzj506cOME777zDyZMnqV27ttNxQpYdnjvkxIkTvP7669x666106tTJ6TjGj6kqu3btYtmyZfTp08fuqnCYFU0HuFwuxo0bx5kzZ+yw3BQqKyuLsmXLEhcXx/Dhw2natKnTkUKe10VTRKqoarovw4SKF198kS+//JK33nqLdu3aOR3H+CGXy8Xnn39ORkYGo0eP5u6773Y6kvEo8pymiPQUkR+AXZ7pDiLyrs+TBSFVZfr06UydOpVx48bx0EMPOR3J+KktW7Zw+vRpRowY4XQUcwFvLgS9AQwEjgOo6jagty9DBaO0tDTGjh3LpEmTuOmmm3jvvffs6qcpkMvlYsOGDQwaNMguEPohr66eq+qhC2bl+iBL0FJVRo0axSeffMLUqVNZunSp9XloCuRyuVBVJk2aRL169ZyOYwrgzTnNQyLSE1ARKQ9MxnOobryzYMECFi9ezKuvvspjjz3mdBzjp1SVxYsXc8UVV9CnTx+n45iL8KaleT/wEHAVkAR0BB70YaagkpqayiOPPMK1117LlClTnI5j/JSq8vXXX5OamkqPHj2cjmMK4U1Ls4Wqjso/Q0SuB9b6JlJweeKJJzh27BhLliyhbFm7w8sULD4+np9++onRo0fbCJJ+zpvf4reAa72YZy6wefNm3n//fR577DGuvdZ+XKZgp06dokWLFjRr1swu/ASAixZNEekB9ATqiMj/5HurGmADkXjhyy+/JCwsjKefftrpKMZPrVu3jri4OO677z4rmAGisHOa5YFw3IW1ar6vX4A7fB8t8EVHR9OpUyd77M0UaO/evXz//feMGDHCbj8LIBdtaarqd8B3IjJTVRNLMVNQyMrKYsOGDTzwwANORzF+6j//+Q/Dhg2jevXqTkcxxeDNOc0zIvIq0AbIu7lQVW/0WaogsHnzZjIzM+nd254DML+lqtx77712SB6AvLnlaDawG2gCTAUOAJt8mCkoLFu2DIAbbrjB4STG3xw6dIhPP/3UCmaA8qZo1lLVGUC2qn6nqhOA63ycK6ClpaXx7rvvMnToUOv30JwnOzubRYsW0aFDB6ejmEvkTdHM9nw/KiJDRKQT0MCblYvIzSISLyJ7ReTPF1mmr4jEishOEfnOy9x+7dVXX+XEiRM8++yzTkcxfmblypXUr1+fVq1aOR3FXCJvzmm+LCLVgUdx359ZDZhS1IdEpAzwDnAT7ieJNonIYlX9Id8yNYB3gZtV9aCIBPygJ4mJifztb39j5MiRNqKk+Y3GjRvToIFXbQ7jp4osmqr6leflKaAf5D0RVJRuwF5V3e/5zDzgFuCHfMvcA3yuqgc920r2Prr/UVX++Mc/IiK88sorTscxfiYpKYlrrrmGMmXsNudAdtHDcxEpIyIjReQxEWnrmTdURNYBb3ux7quA/L0jJXnm5dccuEJEVotIjIiMKWZ+v/K3v/2NL774ghdffJGrr77a6TjGjyQnJzN79mzS060f70BXWEtzBnA1sBF4U0QSgR7An1V1kRfrLuhuXS1g+52B/kAlYL2IbFDVhPNWJHIfcB9Aw4YNvdh06Vu+fDlPPvkkI0aM4NFHH3U6jvETqsqcOXM4evQo/fv3p1q1ak5HMpepsKLZBWivqi4RqQgcA65R1Z+8XHcS7qJ7TgPgSAHLHPMMo5EuItFAB+C8oqmq04HpAF26dLmw8PqFN998k4YNGzJjxgx7uiPEqSrbt2/np59+YuDAgfTp04d69epZhy1BorCr52dV1QWgqplAQjEKJrjv5YwUkSaefjhHAIsvWOYLoJeIlBWRykB3ArCvztzcXNasWcPNN99M5cqVnY5jHJSamsrHH3/M+vXradu2LQANGjSwghlECvuXbCki2z2vBWjmmRZAVbV9YStW1RwReRhYjruDjw9VdaeI3O95f5qq7hKRZcB2wAV8oKo7LnOfSt327dv55Zdf7Okfw+7du2nWrBk9evQgLMyrgRFMgCmsaF72jWSqugRYcsG8aRdMvwq8ernbclJ0dDQAvXr1cjiJccqePXsArAPhEHDRP4WqmljYV2mG9HfR0dE0adLErpiHqJSUFKKiomzcpxBhxw+XSVWJjo62VmYIW7FiBb1797Y/miHCiuZlWr58OceOHbOBsELU2bNnqVKlCl27dnU6iiklXhVNEakkIi18HSbQJCcnM378eFq1asWIESOcjmNKmcvlIjc3l+HDh9tTPiGkyKIpIsOAWGCZZ7qjiFx461DIUVXGjx9Pamoq8+bNs1uNQtCWLVtYvDjkfxVCjjctzRdwP0d+EkBVY4HGvgoUKPbt28eSJUt4/vnnad++0LuvTBBKTExk5cqVdlomBHlTNHNU9ZTPkwSYH3/8EYCePXs6nMSUtnN9Yt5+++3Uq1fP6TimlHnzmMIOEbkHKCMikcBkYJ1vY/m/xET3XVeNGjVyOIkpTZmZmVSoUIEHHnjAxicPUd60NB/BPT5QFjAHdxdxU3yYKSAkJiYSFhbGVVdd2HGTCVa5ubl89NFH7NmzxwpmCPOmpdlCVZ8GbPDufBITE7nqqqtsnJcQsnLlSsLDw4mMjHQ6inGQNy3N10Vkt4i8JCJtfJ4oQMTFxdGihd2FFSoOHz5MXFwcw4cPt16sQlyRRVNV+wF9gRRguojEicgzvg7mz06fPs327dvtIlAIqV+/PhMmTLBby4x3N7er6k+q+iZwP+57Np/zZSh/t2HDBlwuF9df782oHybQJSQksGfPHmrUqOF0FOMHvLm5vZWIvCAiO3APc7EOL0ejDFZr164lLCyM666zkYyDXW5uLsuWLbNz1yaPNxeC/gXMBQao6oU9r4ekNWvW0K5dOxu6IEipKi6XCxFhyZIl1KxZkyZNmjgdy/gJb85pXqeq/7CC6ZaTk8P3339vh+ZBKi0tjffee4+NGzcSFhZG5cqVGTp0qNOxjB+5aEtTROar6l0iEsf5A6J51XN7sDpx4gRpaWm0anXZfTQbP5OWlsbMmTNp165dXmfC/fv3dziV8TeFHZ7/0fPd/szmk5WVBWAdzgaJzMxMdu7cSceOHdm1axft2rWz58lNoS5aNFX1qOflg6r6RP73ROQV4Inffir4ZWZmAlChQgWHk5iSsGzZMn755RdatWplfWIar3hzy9FNBcwbVNJBAsW5lqYVzcCXmppKQkICd911l91/abxW2DnNB4AHgab5RqUEqAqs9XUwf2WH58Hjl19+oU+fPvZvaYqlsHOac4ClwF+BP+ebf1pVT/g0lR+zlmZwyM7OpmHDhtZLlSm2wg7PVVUPAA8Bp/N9ISI1fR/NP1nRDA4rVqxg7dqQPWAyl6GoluZQIAb3LUf5eylQoKkPc/mt1atXIyI0a9bM6SjmErhcLpYvX86PP/7I+PHjnY5jAlBhV8+Her7boxAeOTk5zJgxg4EDB9pwrQEqISGBlJQUJk6caOcyzSUp8jFKEbkeiFXVdBG5F7gW+LuqHvR5Oj+zdOlSDh8+zNtvv+10FFMMWVlZbNiwgZo1a9K2bVuaN29OWJiNXm0ujTf/c94DzohIB+BxIBH42Kep/NT06dOJiIhgyJAhTkcxXtqxYwdvvvkmx48f56qrrkJErGCay+JNhx05qqoicgvwD1WdISJjfR3M3xw6dIglS5bw5JNPWo83AUJVOXDgAPfeey8RERFOxzFBwpuieVpEngRGA71EpAwQclXjk08+weVyMXHiRKejmCKoKmvXrqVNmzbW2YYpcd4cp9yNe1C1Car6E3AV8KpPU/mhzz//nO7du9O4cWOno5girFq1ip07d9rgZ8YnvOka7idgNlBdRIYCmar6kc+T+ZGDBw+yefNmbr31VqejmCJs27aNuLg47r33XqpUqeJ0HBOEvOm5/S5gI3AncBfwvYjc4etg/mTRokUAVjQDQI0aNbjnnnusYBqf8eac5tNAV1VNBhCROsC/gQW+DOZPoqKiaN26Nc2bN3c6irlASkoKiYmJHDlyhKpVq9KvXz+nI5kg5805zbBzBdPjuJefCwqpqan85z//Yfjw4U5HMRfIzc0lKiqKw4cPU69ePVq3bu10JBMCvGlpLhOR5bjHCQL3haElvovkX5YuXUpubi7Dhg1zOorxUFW2bt1Khw4d+MMf/mDjkJtSVWTRVNU/ichtwA24nz+frqpRPk/mJ7788kvq1q1Lt27dnI5icD/KunjxYk6cOEHLli2tH0xT6grrTzMSeA1oBsQBj6nq4dIK5g9UlZUrVzJgwAB7isQPuFwu5s+fT1hYGGPHjrWHDIwjCmtpfgh8BEQDw4C3gNtKI5S/SEhIIDk52caM8RMiQocOHWjZsiVlypRxOo4JUYUVzaqq+r7ndbyIbCmNQP4kOjoagN69ezucJLSda/E3a9aMNm3aOB3HhLjCimZFEenEr/1oVso/rapBX0Sjo6O58soriYyMdDpKSIuOjiYhIYGePXs6HcWYQovmUeD1fNM/5ZtW4EZfhfIH2dnZrFy5kl69etnVWQdt2bKFbdu2MWHCBCpVquR0HGMK7YQ4pO8Snj59OkeOHGHcuHFORwlpzZo1o2nTpoSHhzsdxRgghG5SL47Tp08zdepU+vTpw+DBg52OE5LS0tL4/PPPqVq1KjVq1HA6jjF5fFo0ReRmEYkXkb0i8udClusqIrn+8kz766+/TkpKCn/729/s0NwBqsqiRYuoUaOG3epl/I7P/kd6+t18BxgEtAZGishvnnPzLPcKsNxXWYpr4cKF3HjjjXZDu0PWrVtHVlYWffv2dTqKMb/hTS9HIiL3ishznumGIuJNNekG7FXV/ap6FpgH3FLAco8AC4HkAt4rdampqezYscN+YR0UHh7O7bffbq1M45e8+V/5LtADGOmZPo27BVmUq4BD+aaTPPPyiMhVwK3ANC/WVyrWr1+PqnLDDTc4HSWk5OTkEBUVxZ49e+jQoYOdxzR+y5sOO7qr6rUishVAVVNFxJsusQs6GagXTP8deEJVcws7dygi9wH3ATRs2NCLTV+6NWvWULZsWbp37+7T7ZhfnTp1ioULF1KlShXrGd/4PW+KZrbnvKNCXn+aLi8+lwTkHxy8AXDkgmW6APM8BbM2MFhEclR1Uf6FVHU6MB2gS5cuFxbeErVmzRo6d+5sHUGUoiVLlhAZGckNN9xgF96M3/Pm8PxNIAqoKyL/D1gD/MWLz20CIkWkiadlOgJYnH8BVW2iqo1VtTHuTo0fvLBglqasrCw2btxoh+alIDU1lcWLF5ORkcGIESPsIQITMLzpGm62iMQA/XEfcg9X1V1efC5HRB7GfVW8DPChqu4Ukfs97/vNecxzNm/eTFZWlhVNH0pLS+O7775jx44ddO/enXLlylmxNAGlyKIpIg2BM8CX+eep6sGiPquqS7igw+KLFUtVHVfU+nxtzZo1AFx//fUOJwk+mZmZqCoZGRmULVuWRx55xE6BmIDkzTnNr3GfzxSgItAEiAeCrruZNWvW0KJFC+rUqeN0lKDhcrnYunUrq1atYsCAAbRv356BAwc6HcuYS+bN4Xm7/NMici0wyWeJHHLw4EGWL1/OAw884HSUoOFyufjwww8JCwtj1KhRREREOB3JmMvmTUvzPKq6RUS6+iKMk/76178C8OijjzqcJPBlZ2eTkJBAmzZtGDZsGHXr1rXzliZoeHNO83/yTYYB1wIpPkvkgEOHDjFjxgx+//vf+/w+0GB35MgRoqKiuPLKK2nVqhVXXnml05GMKVHetDSr5nudg/sc50LfxHHGuVbmk08+6XCSwJaUlMTcuXO5+eabadeuXdEfMCYAFVo0PTe1h6vqn0opT6lLTk5mxowZTJgwwVqZlyEjI4OIiAgmTJhArVq1nI5jjM8UNhplWc+9lteWZqDSNmPGDM6ePcuUKVOcjhJwVJV9+/axceNGMjMzGT9+vBVME/QKa2luxH3+MlZEFgOfAenn3lTVz32czedyc3OZNm0aN954Iy1btnQ6TsCJiori559/pnv37rRr184u9piQ4M05zZrAcdxjAp27X1OBgC+aS5Ys4eDBg7zxxhtORwlIt9xyC2FhYVYsTUgprGjW9Vw538GvxfIcn3aaUVoWLVpEzZo1+d3vfud0lIByrmf1/v37U61aNafjGFOqCiuaZYBwvOviLSBt3ryZrl27UrZssW9XDVlnz55lw4YNHD16lKpVqxb9AWOCTKFD+Krqi6WWpJRlZGSwc+dOhg0b5nSUgOByuQgLC2PevHlUqFCBO+64ww7LTUgqrGgG9W/Etm3byM3NpUuXLk5H8WuqypYtW9iyZQsTJ07knnvusZa5CWmF/e/vX2opHLBp0yYAOnfu7HAS/3X69GkWL15MWloat912GyJiBdOEvIv+BqjqidIMUppycnJ45513aN26NQ0aNHA6jt86deoU9evXp3fv3pQpU8bpOMb4hZBsNsyYMYP4+HgWLVpk5+UKsHfvXg4ePMiNN95of1SMuUDIjZGalpbG888/zw033GC3GhUgLi6OqKgoIiMjnY5ijF8KuZbm8uXL+fnnn5kzZ461Mi+QkJDAN998w9ixY6lbt67TcYzxSyFXNLdt20ZYWBg9evRwOorfiYyM5P7777dhKIwpRMgdnm/bto0WLVpQqVIlp6P4lRUrVvDTTz9ZwTSmCCFXNOPi4qyvxwskJiayY8cO66HIGC+EVNHMysoiMTHRejTKR1VZtmwZAwYMoHz58k7HMcbvhVTR3L9/Py6Xy64M55ORkUGDBg1o0yboBhc1xidC6kLQnj17AKxoeuTk5BAWFsaQIUOcjmJMwAiplqYVzV+5XC6ioqJYsWKF01GMCSghVzRr1qxJzZo1nY7iKJfLxcKFC8nKymLw4MFOxzEmoIRU0UxISLBWJu5hdnNzcxkxYoR1wGFMMYVU0dyzZw/Nmzd3OoajUlJSaNCggRVMYy5RyBTN1NRUkpKSQrZoqiqbNm1i1qxZpKenF/0BY0yBQqap8dVXXwFw0003OZyk9J09e5aoqChOnjzJ+PHjqVKlitORjAlYIVM0o6KiqF+/Pl27dnU6SqnKycmhbNmyNGrUiNtvv90OyY25TCFxeH7mzBmWLVvG8OHDCQsLiV0GYMuWLbz//vsAXHfddVYwjSkBIfFb9O9//5uMjAxuvfVWp6OUipycHJYuXcrBgwe5++67Q+oPhTG+FhJFc8eOHQAh0x1camoq2dnZTJw4kQoVKjgdx5igEhJF89ChQ9SsWTMkLoCcOHGC8PBwbrvtNqejGBOUQuK47dChQ1x99dVOxygVy5YtY+/evU7HMCZoWdEMIue6vrOnnozxHSuaQWTv3r1cffXVVKxY0ekoxgStoD+nmZ6eTmpqatAXTVWlcePGVjCN8bGgb2keOnQIgIYNGzqcxHdiYmL49NNPqVKlCs2aNXM6jjFBLehbmueKZjC2NFWVpUuXsn//fkaOHOl0HGNCQtAXzX379gHQqFEjh5OUvMTERJKTk5k4caIdlhtTSoK+aK5Zs4Z69eoF1eG5qpKUlETjxo0ZM2aMPfFjTCkK6t82VeW7776jd+/eiIjTcUrM3r17Wbx4MapqBdOYUubT3zgRuVlE4kVkr4j8uYD3R4nIds/XOhHpUJLbT0xMJCkpid69e5fkah2lqqxcuZJ+/foF1R8CYwKFz4qmiJQB3gEGAa2BkSLS+oLFfgT6qGp74CVgeklmiI6OBgiqopmQkABAq1atHE5iTGjy5TnNbsBeVd0PICLzgFuAH84toKrr8i2/AWhQkgG2bdtGpUqVgmZMb1UlMjKSevXqWSvTGIf48vD8KuBQvukkz7yL+T2wtKA3ROQ+EdksIptTUlK8DpCWlka1atWC4rxfWloas2bN4vTp01SvXt3pOMaELF9Wk4KaQlrggiL9cBfNJwp6X1Wnq2oXVe1Sp04drwOcOXMmKHo2Sk1N5cMPP6Rx48ZUq1bN6TjGhDRfHp4nAfnvKG8AHLlwIRFpD3wADFLV4yUZID09ncqVK5fkKkudqjJ//nyuu+46unXr5nQcY0KeL4vmJiBSRJoAh4ERwD35FxCRhsDnwGhVTSjpAGfOnAnoopmcnEzt2rUZN26cdSZsjJ/w2eG5quYADwPLgV3AfFXdKSL3i8j9nsWeA2oB74pIrIhsLskMgXx4npCQwMyZM0lJSbGCaYwf8ekTQaq6BFhywbxp+V5PBCb6avvp6elERET4avU+oaqsWrWKrVu3cs8993DllVc6Hcn4uezsbJKSksjMzHQ6SkCoWLEiDRo0oFy5cpf0+aB+jDLQDs/PDbdbpUoVJk2aRHh4uNORTABISkqiatWqNG7c2G5FK4Kqcvz4cZKSkmjSpMklrSPw78UpRCAVzV9++YV3332XM2fO0L17dyuYxmuZmZnUqlXLCqYXRIRatWpdVqs8qItmenp6QJzTPHPmDHPnzuXaa68NmCJv/IsVTO9d7s8qqItmRkZGQHSZtmDBApo1a8b111/vdBRjLomIMHr06LzpnJwc6tSpw9ChQ4u1nsaNG3Ps2LHLXsaXgvqcZlZWlt8WTVUlNjaWdu3aMWLECMqXL+90JGMuWZUqVdixYwcZGRlUqlSJb775hquuKuwBwMAVtC3N3NxccnNz/fJ2nYyMDObOnUtMTAyZmZlWME1QGDRoEF9//TUAc+fOPW80gRMnTjB8+HDat2/Pddddx/bt2wE4fvw4AwYMoFOnTkyaNAnVXx8a/OSTT+jWrRsdO3Zk0qRJ5Obmlu4OXUTQtjSzsrIA/K5onj17lunTp9OyZUvuvvtuypQp43QkE0SmTJlCbGxsia6zY8eO/P3vfy9yuREjRvDiiy8ydOhQtm/fzoQJE/jPf/4DwPPPP0+nTp1YtGgRK1euZMyYMcTGxjJ16lRuuOEGnnvuOb7++mumT3d3dLZr1y4+/fRT1q5dS7ly5XjwwQeZPXs2Y8aMKdF9uxRWNEuJy+Xi+PHj1KlTh5EjR1K3bl2nIxlTotq3b8+BAweYO3cugwcPPu+9NWvWsHDhQgBuvPFGjh8/zqlTp4iOjubzzz8HYMiQIVxxxRUAfPvtt8TExNC1a1fAfXTmL78zVjR9bPXq1ezcuZPU1FTq1q3LhAkT/OYf3wQfb1qEvvS73/2Oxx57jNWrV3P8+K9dSeQ/7D7n3FXsgq5mqypjx47lr3/9q+/CXqKgPad57j4sp4rmjh07yMzMpEWLFtxxxx088cQT3HfffZQtG7R/p4xhwoQJPPfcc7Rr1+68+b1792b27NmAuyFRu3ZtqlWrdt78pUuXkpqaCkD//v1ZsGABycnJgPucaGJiYinuycUF7W/wihUrAGjd+sLO4n1v48aNrFu3jquvvjrgHuM05nI0aNCAP/7xj7+Z/8ILLzB+/Hjat29P5cqVmTVrFuA+1zly5EiuvfZa+vTpkzcAYuvWrXn55ZcZMGAALpeLcuXK8c477/jFqLJSULPZn3Xp0kU3bz6/X4/Vq1fTt2/fvGlVpXPnzuTm5hIbG1uqN/5u3ryZNWvWMG7cOGrUqFFq2zWha9euXTb8STEV9DMTkRhV7VLUZ4Py8Hzjxo1s3bqV+++/v9SflChbtixjx461gmlMkArKw/Np06YRHh7OvffeW2rbjImJoVKlSnTs2LHUtmmMKX1B2dL85ptvGDZsGFWrVvX5tg4ePMjSpUuJjo6mXr16Pt+eMcZZQVk009PTqV27ts/Wf/r0aTZu3AjA7t27qVixIhMmTKBmzZo+26Yxxj8E5eF5ZmamT585X7FiRV7vSQMGDPDZdowx/ifoWpqqSmZmJpUqVfLJ+g8ePMjBgwe58cYbfbJ+Y4x/C7qiee5JIF+1NA8cOMBNN91knWwYk0+ZMmXo2LEjbdu2ZdiwYZw8ebJE1jtz5kwefvjhEllXSQm6onnuSSBftDRzcnLo3bs3bdu2LfF1GxPIKlWqRGxsLDt27KBmzZq88847TkfymaArmueed61evXqJrjczM5O3336btLS0El2vMcGmR48eHD58GHDfM92zZ086depEz549iY+PB9wtyNtuu42bb76ZyMhIHn/88bzP/+tf/6J58+b06dOHtWvX5s1PTEykf//+tG/fnv79+3Pw4EEAxo0bxwMPPEC/fv1o2rQp3333HRMmTKBVq1aMGzeuxPcv6Irmnj17AIiMjCyxdaoqX3zxBZGRkTZ2jzGFyM3N5dtvv+V3v/sdAC1btiQ6OpqtW7fy4osv8tRTT+UtGxsby6effkpcXByffvophw4d4ujRozz//POsXbuWb775hh9++CFv+YcffpgxY8awfft2Ro0axeTJk/PeS01NZeXKlbzxxhsMGzaM//7v/2bnzp3ExcWVeFd5QXf13BdFc926dZw5c4bbb7+9xNZpjK+sXr2a1atX503fd999AHl9VQL07duXvn378n//93+cPn0agIiICCZNmsSXX35JTExM3rKPPvpokfc8Z2Rk0LFjRw4cOEDnzp256aabADh16hRjx45lz549iAjZ2dl5n+nfv3/eEWHr1q1JTEzk2LFj9O3blzp16gBw9913k5CQAMD69evzupEbPXr0ea3TYcOGISK0a9eOK6+8Mq/DkDZt2nDgwIESfegk6IpmQkICVatWLbHu11SVTp060alTJ+uhyASEcwXxQi+88MJv5j366KO/mTds2DCGDRtWrG2eO6d56tQphg4dyjvvvMPkyZN59tln6devH1FRURw4cOC8XPl7ICtTpgw5OTmA9wOf5V/u3LrCwsLOW29YWFjeektKUB6eR0ZGXvIz56rKihUrmD17Nm+99RZz5syhcuXKNkqkMV6oXr06b775Jq+99hrZ2dmcOnUqb6ygmTNnFvn57t275/XFmZ2dzWeffZb3Xs+ePZk3bx4As2fP5oYbbvDJPhQl6JpOe/bsyevtubhycnIoW7YstWrVolGjRtSsWTOvJ2ljjHc6depEhw4dmDdvHo8//jhjx47l9ddf9+re5oiICF544QV69OhBREQE1157bd7YQG+++SYTJkzg1VdfpU6dOvzrX//y9a4UKKi6hjt79iyVKlXi6aef5sUXXyzWelWVWbNm0bt3b5o2bVqSkY3xKesarvisaziPH3/8EZfLdUkXgbZv387Zs2dp3LhxyQczxgSNoCqaR44cAdy9RxdHZmYm//73vxkyZAhhYUH1IzHGlLCgqhDnTjUUZ1hcl8tF2bJlGTZsWNAObm+MKTlBVTTT09MBKFeuXJHLqirbtm3jrbfeIisri+bNm/s6njEmCATV1fNz4wG1adOm0OWOHz9OVFQULpeL2267La+bN2OMKUpQFc3NmzfTvHlzqlWrVuhyZcuWpUePHrRu3brUxxAyxgS2oDo8j4mJoUuXwu8YOHPmDDk5ObRp08YKpjEl5GJ9MowbN44FCxZc0jpfeOEFXnvttcuJ5RNBUzR/+uknDh8+XGTR3Lx5M+vWrSulVMaYYBM0RXPTpk0AhRZNl8vlVWvUGHNpVJWHH36Y1q1bM2TIEJKTk/Pei4mJoU+fPnTu3JmBAwdy9OhRAN5//326du1Khw4duP322zlz5oxT8b0SNEVz3bp1lCtXjs6dO190mejoaGrXrk1EREQpJjMmdERFRREfH09cXBzvv/9+3lFddnY2jzzyCAsWLCAmJoYJEybw9NNPA3DbbbexadMmtm3bRqtWrZgxY4aTu1CkoLkQtHbtWq699toCe2zfvXs3DRo0oFu3bvTq1cuBdMaUntjY2BIbbgKgRo0aXnetFh0dzciRIylTpgz169fPe948Pj6eHTt25HUZl5ubm9d42bFjB8888wwnT54kLS2NgQMHllh2XwiKopmdnc2mTZt44IEHfjN/+fLl7Nu3jxEjRnDllVc6lNCY0lOSfUdeioIusKoqbdq0Yf369b95b9y4cSxatIgOHTowc+bM8/oC9UdBcXi+Z88eMjMz6dGjR948VWXmzJlkZWUxadIkK5jGlILevXszb948cnNzOXr0KKtWrQKgRYsWpKSk5BXN7Oxsdu7cCcDp06eJiIggOzub2bNnO5bdW0HR0oyLiwPIO/Q+ffo04eHh3H777VxxxRV2a5ExpeTWW29l5cqVtGvXLm+cH4Dy5cuzYMECJk+ezKlTp8jJyWHKlCm0adOGl156ie7du9OoUSPatWuX15O8vwqKruF69uzJ8ePHiY+PJz09nWnTpnHvvfda69KEBOsarvhCvmu4uLg4evXqlTcAWocOHaxgGmN8wqdFU0RuFpF4EdkrIn8u4H0RkTc9728XkWsvZTvp6elERESwa9cu0tPT6dev3+WHN8aYAvjsnKaIlAHeAW4CkoBNIrJYVX/It9ggINLz1R14z/O92FwuF61ataJx48bF6hrOGGOKw5ctzW7AXlXdr6pngXnALRcscwvwkbptAGqISLHuPD958iSqyvbt28nNzbUB0ExICrRrE0663J+VL4vmVcChfNNJnnnFXaZQiYmJAPzXf/2XDbFrQlLFihU5fvy4FU4vqCrHjx+nYsWKl7wOX1aZgu7zufBf1ZtlEJH7gPsAGjZseN57jRo1YsGCBVx33XWXGNOYwNagQQOSkpJISUlxOkpAqFixYrGHxMnPl0UzCbg633QD4MglLIOqTgemg/uWo/zv1ahRg9tvv70k8hoTkMqVK0eTJk2cjhEyfHl4vgmIFJEmIlIeGAEsvmCZxcAYz1X064BTqnrUh5mMMeay+Kylqao5IvIwsBwoA3yoqjtF5H7P+9OAJcBgYC9wBhjvqzzGGFMSfHrlRFWX4C6M+edNy/dagYd8mcEYY0pSwD1GKSIpQGIBb9UGjpVyHF8Lxn0C269AEoz7BAXvVyNVrVPUBwOuaF6MiGz25rnRQBKM+wS2X4EkGPcJLm+/guLZc2OMKS1WNI0xphiCqWhOdzqADwTjPoHtVyAJxn2Cy9ivoDmnaYwxpSGYWprGGONzAVU0S6t/ztLmxX6N8uzPdhFZJyIdnMhZHEXtU77luopIrojcUZr5LpU3+yUifUUkVkR2ish3pZ3xUnjxf7C6iHwpIts8++X3D6KIyIcikiwiOy7y/qXVC1UNiC/cTxXtA5oC5YFtQOsLlhkMLMXdEch1wPdO5y6h/eoJXOF5Pcjf98ubfcq33ErcD0Dc4XTuEvq3qgH8ADT0TNd1OncJ7ddTwCue13WAE0B5p7MXsV+9gWuBHRd5/5LqRSC1NEulf04HFLlfqrpOVVM9kxtwd2ziz7z5twJ4BFgIJJdmuMvgzX7dA3yuqgcBVDUQ9s2b/VKgqrhHKQzHXTRzSjdm8ahqNO6cF3NJ9SKQimap9M/pgOJm/j3uv47+rMh9EpGrgFuBaQQOb/6tmgNXiMhqEYkRkTGllu7SebNfbwOtcPdCFgf8UVVdpRPPZy6pXgRSr70l1j+nn/E6s4j0w100b/BposvnzT79HXhCVXMDaIhlb/arLNAZ6A9UAtaLyAZVTfB1uMvgzX4NBGKBG4FmwDci8h9V/cXH2XzpkupFIBXNEuuf0894lVlE2gMfAINU9XgpZbtU3uxTF2Cep2DWBgaLSI6qLiqVhJfG2/+Dx1Q1HUgXkWigA+DPRdOb/RoP/K+6TwbuFZEfgZbAxtKJ6BOXVi+cPllbjJO6ZYH9QBN+PVnd5oJlhnD+id2NTucuof1qiLv7vJ5O5y2pfbpg+ZkExoUgb/6tWgHfepatDOwA2jqdvQT26z3gBc/rK4HDQG2ns3uxb425+IWgS6oXAdPS1CDtn9PL/XoOqAW862mZ5agfd6Lg5T4FHG/2S1V3icgyYDvgAj5Q1QJvefEXXv57vQTMFJE43EXmCVX1696PRGQu0BeoLSJJwPNAObi8emFPBBljTDEE0tVzY4xxnBVNY4wpBiuaxhhTDFY0jTGmGKxoGmNMMVjRNF7x9EQUm++rcSHLppXA9maKyI+ebW0RkR6XsI4PRKS15/VTF7y37nIzetZz7ueyw9MLUI0ilu8oIoNLYtvGGXbLkfGKiKSpanhJL1vIOmYCX6nqAhEZALymqu0vY32Xnamo9YrILCBBVf9fIcuPA7qo6sMlncWUDmtpmksiIuEi8q2nFRgnIr/pxUhEIkQkOl9LrJdn/gARWe/57GciUlQxiwau8Xz2fzzr2iEiUzzzqojI156+HneIyN2e+atFpIuI/C9QyZNjtue9NM/3T/O3/Dwt3NtFpIyIvCoimzx9LU7y4seyHk+HDyLSTdx9n271fG8hIuWBF4G7PVnu9mT/0LOdrQX9HI2fcfoxJ/sKjC8gF3eHDbFAFO5H76p53quN+6mKc0cuaZ7vjwJPe16XAap6lo0GqnjmPwE8V8D2ZuJ5tBK4E/ged0cYcUAV3N2T7QQ6AbcD7+f7bHXP99W4W3V5mfItcy7jrcAsz+vyuHu9qQTcBzzjmV8B2Aw0KSBnWr79+wy42TNdDSjref1fwELP63HA2/k+/xfgXs/rGrifUa/i9L+3fV38K2AeozSOy1DVjucmRKQc8BcR6Y37ccGrcD+T/FO+z2wCPvQsu0hVY0WkD9AaWOt5JLQ87hZaQV4VkWeAFNy9O/UHotTdGQYi8jnQC1gGvCYir+A+pP9PMfZrKfCmiFQAbgaiVTXDc0qgvfzao3x1IBL48YLPVxKRWNzPOMcA3+RbfpaIROLuOafcRbY/APidiDzmma6Iu6+BXcXYB1OKrGiaSzUKdw/enVU1W0QO4P6Fz6Oq0Z6iOgT4WEReBVKBb1R1pBfb+JOqLjg3ISL/VdBCqpogIp1xP0f8VxFZoaoverMTqpopIqtxd312NzD33OaAR1R1eRGryFDVjiJSHfgKeAh4E/ez2qtU9VbPRbPVF/m8ALerarw3eY3z7JymuVTVgWRPwewHNLpwARFp5FnmfWAG7qEHNgDXi8i5c5SVRaS5l9uMBoZ7PlMF96H1f0SkPnBGVT8BXvNs50LZnhZvQebh7qyhF+5OK/B8f+DcZ0SkuWebBVLVU8Bk4DHPZ6rj7gkI3Ifk55zGfZrinOXAI+JpdotIp4ttw/gHK5rmUs0GuojIZtytzt0FLNMXiBWRrbjPO/5DVVNwF5G5IrIddxFt6c0GVXUL7nOdG3Gf4/xAVbcC7YCNnsPkp4GXC/j4dGD7uQtBF1iBezyZf6t7uAdw9136A7BF3ANz/ZMijsw8WbYBI4C/4W71rsV9vvOcVUDrcxeCcLdIy3my7fBMGz9mtxwZY0wxWEvTGGOKwYqmMcYUgxVNY4wpBiuaxhhTDFY0jTGmGKxoGmNMMVjRNMaYYrCiaYwxxfD/AV3X+peWQrUaAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(df_scores.fpr, df_scores.tpr, color='black', label='Model')\n",
    "plt.plot(df_rand.fpr, df_rand.tpr, color='black', lw=1,\n",
    "         linestyle='dashed', alpha=0.5, label='Random')\n",
    "plt.plot(df_ideal.fpr, df_ideal.tpr, color='black', lw=0.5,\n",
    "         linestyle='solid', alpha=0.5, label='Ideal')\n",
    "\n",
    "plt.legend()\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "\n",
    "# plt.savefig('04_roc_curve_with_baselines.svg')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA6QUlEQVR4nO3deXxU9dn38c8VtoSwCQTZZFO2oIgSUFBcUFllE4K71dZWW7X1ftzq0lraPr1vpbe21ipF8VFbLbIIAgZQFIxBlgTCvske1iQkINkzmev5YwaMaZYJZObMcr1fr3llzsxvzvmeLFd+Z/sdUVWMMcb4JsrpAMYYE0qsaBpjTC1Y0TTGmFqwommMMbVgRdMYY2rBiqYxxtSCFU1jjKkFK5rG70Rkv4gUikieiBwTkXdFpEmFNoNF5EsROS0ip0RkoYjEV2jTTET+IiIHvfPa7Z1uHdg1MpHMiqYJlDGq2gToB1wBPHvmDREZBHwGfAK0B7oCG4GVItLN26Yh8AXQBxgBNAMGAyeAgf4KLSL1/TVvE5qsaJqAUtVjwFI8xfOMl4H3VfWvqnpaVXNU9QVgNfA7b5v7gE7ABFXdpqpuVc1U1T+oalJlyxKRPiLyuYjkiMhxEXnO+/q7IvLHcu1uEJFD5ab3i8gzIrIJyBeRF0RkToV5/1VEXvM+by4iM0TkqIgcFpE/iki98/tOmWBlRdMElIh0BEYCu73TjfH0GGdX0nwWcIv3+c3AElXN83E5TYFlwBI8vddL8PRUfXUnMBpoAfwTGCUizbzzrgdMBj70tn0PcHmXcQUwDHiwFssyIcSKpgmU+SJyGsgAMoEXva+3xPN7eLSSzxwFzuyvbFVFm6rcChxT1f9V1SJvD3ZNLT7/mqpmqGqhqh4A1gPjve8NBQpUdbWIXIjnn8DjqpqvqpnAq8AdtViWCSFWNE2gjFfVpsANQC++L4a5gBtoV8ln2gHZ3ucnqmhTlYuAPeeU1COjwvSHeHqfAHfxfS+zM9AAOCoiJ0XkJPAPoM15LNsEMSuaJqBU9SvgXeDP3ul8YBWQWEnzyXy/Sb0MGC4isT4uKgO4uIr38oHG5abbVha1wvRs4Abv7oUJfF80M4BioLWqtvA+mqlqHx9zmhBjRdM44S/ALSLSzzv9a+BHIvJLEWkqIhd4D9QMAqZ42/wTT4GaKyK9RCRKRFqJyHMiMqqSZSwC2orI4yLSyDvfq7zvbcCzj7KliLQFHq8psKpmASuA/wfsU9Xt3teP4jny/7/eU6KiRORiEbm+lt8TEyKsaJqA8xag94HfeKdTgOHAbXj2Wx7Ac0DlWlX91tumGM/BoB3A58B3wFo8m/n/sa9SVU/jOYg0BjgGfAvc6H37n3hOadqPp+B95GP0D70ZPqzw+n1AQ2Abnt0Nc6jdrgQTQsQGITbGGN9ZT9MYY2rBiqYxxtSCFU1jjKkFK5rGGFMLVjSNMaYWQm4El9atW2uXLl2cjmGMCTPr1q3LVtW4mtr5rWiKyDt4rv/NVNVLK3lfgL8Co4AC4H5VXV/TfLt06UJaWlpdxzXGRDgROeBLO39unr+LZ9zDqowEunsfPwPe9GMWY4ypE34rmqqaDORU02QcnjEUVVVXAy1ExK6iMMYENSf3aXbghyPJHPK+Vpvhv4wxhtLSUvLz88nPzycvL6/GrwkJCYwbN+6cluVk0ZRKXqv0mk4R+RmeTXg6derkz0zGGD9RVYqLi30ubL58PfO8pKSkVlkee+yxkCyah/CMeXhGR+BIZQ1VdTowHSAhIcEuljfGz1SVwsJC8vLyzj5Onz79g+mqXqvq9fz8fMrKynzOUL9+fZo0aUJsbOwPvsbFxdG1a9f/eL2mry6Xiy+//JJJkybRsmXLc/7eOFk0FwCPishM4CrglHeYLWOMn6gqmZmZ7Nu3j7179/7g6+HDh39Q8HwdzCcqKoomTZr84NG0aVPatWt3tmg1bdq01kWuYcOGdbruS5YsITExkQsuuOC85uPPU47+jWeU7tbem1a9iGeEa1R1GpCE53Sj3XhOOXrAX1mMCSclJSXs2bOHkydPVtlGVcnOzq60OBYUFPygbdu2benatSv9+vWjWbNm/1EAzxTBql6Pjo7GcwZhcDpx4gQHDhxgxIjqTubxnd+KpqreWcP7Cjzir+UbE+pycnLYsWPHDx47d+5kz549tdrMjY2NpVu3blx88cXccsstdOvWja5du9KtWze6dOlC48aNa55JiMrKyiIpKYkJEybU2TxD7oogY0KZqnL8+HFOnDhx9pGTk3P2eVZWFrt372bHjh1kZWWd/VzDhg3p0aMHffv2ZfLkyfTq1YvWrVtXsyRo0aIF3bp1o3Xr1kHdE/SX0tJSysrKmDhxIk2aNKmz+VrRNMbP3G43q1atYs6cOcydO5eMjIr3bPNo2LAhrVq14pJLLmH8+PH06tXr7KNz587Uq2e3UvfV0aNH+eKLL7jrrruIiqrb09GtaBrjB2VlZaSkpJwtlEePHqVRo0YMHz6cp556igsvvJBWrVrRsmXLs19jY2MjskdY1woLC1m2bBkTJ06s84IJVjSNqZWioiIWLFhAXl5epe+rKqmpqcybN4/MzEyio6MZNWoUkyZNYvTo0TRr1izAiSNLRkYGsbGx3H333X4pmGBF0xifuN1uZs6cyXPPPceBA9WP69C4cWNuvfVWJk2axMiRI+t0f5qp2oEDB/j6669JTEz0W8EEK5rG1GjFihU8+eSTrFu3jiuuuIJ//OMf9O7du8r2cXFxxMTEBDChOXP+6eTJk+v8/M6KrGgaU4Vt27bxzDPPsGjRIi666CLef/99v272mXOze/duDh8+zPXXB+ZW81Y0jSmnoKCA5cuXM2fOHN5//32aNGnC//zP//DLX/7Seo9BaN++faxbt46JEycGbJlWNE3E27t3L0lJSSQlJbF8+XKKioqIjY3l0Ucf5Te/+U2N50MaZxw7doy2bdsyceJE6tcPXCmzomki0r59+/jb3/5GUlISO3fuBKBHjx48/PDDjB49miFDhtCoUSOHU5qqbNmyhZ07dzJhwoSA7y6xomkiSllZGa+//jrPPfccLpeLG2+8kV/84heMGjWKSy65xOl4xgfFxcVkZGQ4UjDBiqaJAKrKsWPH2LJlC7/97W9ZvXo1o0eP5s033+Siiy6qeQYmaKSnp9OmTRtGjhzpWAYrmiasHDp0iFWrVrFz586zA1zs2rWL7777DoDWrVvz4Ycfcscdd9jVNyEmLS2NY8eO0a9fP0dzWNE0IU9VSUlJ4a9//Svz5s3D7XYDcNFFF9GrVy/uu+8+evbsSc+ePRk4cCDNmzd3OLGpraKiIi688EL69+/v+D87K5ompC1evJgXXniB9evXc8EFF/DUU08xefJkevbsSWxsrNPxTB1YuXIlLpcrYOdh1sSKpglJx48f5/HHH2fmzJl0796dadOmce+994b12JCR6Ntvv6WoqIihQ4c6HeUsK5om6OXl5bF9+3a2bt3Khg0bWLt2LevXr0dVmTJlCs8884ydHhRmVJX09HT69u3LJZdc4vgmeXlWNE1QUVU++eQTVq5cybZt29i6desPBsiIiYmhf//+PProozz44IP06tXLwbTGH1SVZcuWER0dHdCT1n0VfIlMxMrNzeWnP/0pc+fOpWHDhvTq1YvBgwfz05/+lPj4ePr06UO3bt2C8g/J1A1VpbS0lDZt2nD55Zc7HadS9ttngsKqVau48847OXz4MFOnTuXxxx+34hhhVJXFixfTp0+foC2YADZci3GU2+3mpZdeYsiQIURFRbFy5UqefPJJK5gRaNmyZVx44YV07tzZ6SjVst9M45jMzEzuu+8+li5dSmJiIm+99ZadQxmB3G43hw8fZsiQIURHRzsdp0bW0zSOWL58Of369WPFihVMmzaNjz76yApmBCorK2P+/Pnk5uaGRMEEK5omwFwuF7/73e+46aabaNasGWvXruWhhx4KqlNKTOBkZGTQs2dP+vbt63QUn9nmuQmYgwcPcvfdd5OSksK9997LG2+8YffPiVAul4slS5YwYsSIkNt/bT1NExBz587l8ssvZ8OGDfzzn/88Oyq6iTwul4u5c+fSo0ePkCuYYEXT+Fl+fj4PPfQQkyZNonv37qSnp3PPPfc4Hcs4pKSk5Ow4pj169HA6zjmxomn8Jj09nYSEBN566y2efvppUlJSbKDfCFZcXMzs2bPJzs6mTZs2Tsc5Z1Y0TZ1zu928+uqrXH311Xz33Xd8/vnnvPTSS36/taoJbikpKVx77bV07NjR6SjnJfR2KJiglpWVxb333svSpUsZN24cb7/9tt2YLMIVFhaydetWhg4dGhZnSVhP09SZM5vjK1as4M0332TevHlWMCNcQUEBs2fPpn379mFRMMGKpqkjH374Iddccw1ut5uUlBQefvjhsPkjMefG7XaTn5/PsGHDaN++vdNx6owVTXNeXC4XTz75JHfffTcDBgxg3bp1JCQkOB3LOOz06dN88MEHNG/enLZt2zodp07ZPk1Tox07dnDo0KGz0/n5+ezatYudO3eydu1aNm/ezCOPPMKrr75KgwYNHExqgkFZWRnz5s1j1KhRYXnwz4qmqdKePXt44YUXmDlzZqXvt2nThp49e/Lee+9x3333BTidCUYnT56kpKSEO++8M2z/gVrRND+QmZnJrFmzmDlzJitXriQmJobnn3+e4cOHn91H2ahRI7p3706LFi2cDWuCSk5ODgsWLGD8+PFhWzDBiqYp5+DBg1x++eWcPHmSyy67jD/96U/86Ec/Cqud+MZ/9uzZw4QJE8J+tCormgbwjJr9i1/8gpKSElJTU+1gjvFZVlYW6enpDBs2zOkoAWFHzw0As2fP5tNPP+UPf/iDFUzjs+zsbJKSkhg8eLDTUQJGVNXpDLWSkJCgaWlpTscIK7m5ufTu3ZuOHTuyevXqkBx5xgRebm4uMTExuN3usLjfvIisU9UaewzW0zQ8+eSTZGdn8/bbb1vBND45fPgwixYtQkTComDWhhXNCPfpp5/yzjvv8NRTT9GvXz+n45gQ4Ha7Wbt2LYmJiTRq1MjpOAFnm+cR7MSJE1x66aW0bt2atLS0iPwDMLWzf/9+XC5XWA7xZ5vnpkaPPfYY2dnZvP/++1YwTY327t3LN998Q6dOnZyO4ii/Fk0RGSEiO0Vkt4j8upL3m4vIQhHZKCJbReQBf+Yx35s9ezb//ve/efHFF7niiiucjmOCXGlpKSJCYmJiWF4aWRt+K5oiUg/4OzASiAfuFJH4Cs0eAbap6uXADcD/ikhk/0QCYPfu3Tz44IMMHDiQX//6P/6XGfMDO3fuZPHixXTt2jWsr/TxlT97mgOB3aq6V1VLgJnAuAptFGgqnuvzmgA5gMuPmSJeQUEBt912G/Xr12fWrFl2tNxU69ixY2zatInRo0c7HSVo+LNodgAyyk0f8r5W3utAb+AIsBn4laq6K85IRH4mImkikpaVleWvvGFPVXnooYfYsmULH374IZ07d3Y6kgliu3btokWLFkycOJF69eo5HSdo+LNoVjYCbcVD9cOBDUB7oB/wuog0+48PqU5X1QRVTYiLi6vrnBGhpKSEF154gX/9619MmTKF4cOHOx3JBLENGzawbds2GjZsSFSUHS8uz5/fjUPAReWmO+LpUZb3APCxeuwG9gG9/JgpIq1evZr+/fvzpz/9ifvuu4/nn3/e6UgmiKkqBQUFjB071gpmJfz5HUkFuotIV+/BnTuABRXaHARuAhCRC4GewF4/ZooYbrebpKQkhg0bxqBBg8jNzeWTTz7hvffesz8EU6XU1FS2bNnC4MGD7fekCn77rqiqC3gUWApsB2ap6lYReVhEHvY2+wMwWEQ2A18Az6hqtr8yRYpZs2bRu3dvRo8ezZYtW/jDH/7Atm3bGDt2rNPRTBBbt24d2dnZXHrppU5HCWp2RVCYWb9+PQMHDuTSSy/l6aefZtKkSRF/Xp2pWWZmJo0bNyY2NjZib4jn6xVBdr5JGCktLeUnP/kJcXFxLF++nAsuuMDpSCYEfPXVV5SVlTF06FCno4QEK5ph5M9//jMbNmxg3rx5VjCNT7KyshARK5i1YHt6w8SRI0f44x//yIQJExg/frzTcUyQU1VWrFhB48aNue6665yOE1KsaIaJ5557DpfLxdSpU52OYoKcqrJ06VIaNWpEbGys03FCjm2eh4G0tDTee+89nn76aS6++GKn45ggpqqUlJQQHx8f8aMVnSvraYa4nTt3cvfdd9OmTRs7ad1Uy+12s3DhQvbs2WMF8zxYTzOEHT58mKuuuoqGDRsyd+5cmjX7jytQjTlr9erVdOrUifj4ioONmdqwohnCXn31VfLy8ti2bRs9evRwOo4JUmVlZWzcuJFBgwZF7DmYdck2z0NUbm4u//jHP7jjjjusYJoqlZWVMW/ePKKjo61g1hHraYaov/3tb+Tl5fH00087HcUEKVUlLy+Pvn372j/WOmQ9zRAzc+ZMevbsyYsvvsjo0aPp27ev05FMECotLWXWrFlERUVZwaxjVjRDyOLFi7nnnnto0qQJr7/+Oh988IHTkUwQUlXmzZtHQkICTZs2dTpO2LHN8xCRnp5OYmIiffv2ZcWKFfbHYCpVXFzMyZMnGTt2LNHR0U7HCUvW0wwBJ0+eZMyYMbRs2ZJFixZZwTSVKioqYvbs2RQXF1vB9CPraYaAZ555hqNHj7JmzRrat2/vdBwTpLZv384NN9xAx44dnY4S1qxoBrnPP/+c6dOn88QTT5CQUONQfyYCFRQUkJyczIgRI5yOEhFs8zyI7dixg8TERC699FKmTJnidBwThPLz85kzZw5XXHGF01EihvU0g1RxcTGTJk2iUaNGLFy40EajMf+hoKCA+vXrM3r0aFq1auV0nIhhPc0gNWXKFLZu3cq7775Lly5dnI5jgsx333139qCPFczAsqIZhFJTU3nppZd44IEHGDlypNNxTBD64osvGDNmjA3S4gDbPA8y6enpTJ48mXbt2vHKK684HccEmZycHI4fP8748ePtWnKHWE8zSKgq06ZNY9CgQZSWlvLxxx/TokULp2OZIJKdnc2CBQvo0KGDFUwHWdEMAkVFRdx///38/Oc/58YbbyQ9PZ2BAwc6HcsEEbfbzenTp5k4caJtkjvMimYQeOyxx3j//ff53e9+x6effkpcXJzTkUwQOXbsGLNnz6ZLly52NVgQsKLpsFmzZvH222/z7LPP8uKLLxIVZT8S873Tp0/z2WefMWbMGNskDxKiqk5nqJWEhARNS0tzOkadOHXqFN26daN79+58/fXXNGjQwOlIJogcOXKE5s2b07BhQ/vdCAARWaeqNV52Z90aB/3lL38hJyeHN9980/4ozA8cPHiQL7/8knr16tnvRpCxU44ckpOTwyuvvMKECRPsEjjzH/bv309iYiKNGjVyOoqpwIqmA9xuN/fffz8FBQV2Tbn5gT179pCdnc11113ndBRTBZ83z0XELn6uI7///e9ZuHAhr776KpdddpnTcUyQ2LNnD6mpqVx55ZVORzHVqLFoishgEdkGbPdOXy4ib/g9WRhSVaZPn86UKVO4//77eeSRR5yOZILEyZMnad26NZMmTbJ9mEHOl57mq8Bw4ASAqm4EbNuhlvLy8vjRj37EQw89xC233MKbb75pp5AYALZu3cqyZcto1qwZ9evbHrNg59PmuapmVHipzA9Zwpaqcvfdd/Ovf/2LKVOmsHjxYrsdgQE842Hu3r2bCRMm2D/REOHLv7UMERkMqIg0BH6Jd1Pd+GbOnDksWLCAqVOn8uSTTzodxwSJjRs30qFDB8aNG+d0FFMLvvQ0HwYeAToAh4B+wC/8mCms5Obm8thjj3HllVfy+OOPOx3HBIl169Zx8OBBWrZs6XQUU0u+9DR7qurd5V8QkWuAlf6JFF6eeeYZsrOzSUpKsv1VBgCXy0XTpk259dZbbZM8BPnS0/ybj6+ZCtLS0njrrbf4r//6LzuNxACwatUq1qxZQ48ePaxghqgquz4iMggYDMSJyP8p91YzoJ6/g4WDhQsXEhUVxfPPP+90FBMEtm7dSl5eHjfffLPTUcx5qG57sSHQxNum/HhU3wGT/BkqXCQnJ3PFFVfYYMIRTlXZvn07PXv2JD4+3nqYIa7KoqmqXwFfici7qnoggJnCQnFxMatXr+bnP/+501GMg1SVL7/8kgYNGhAfH+90HFMHfDkyUSAiU4E+wNmTC1V1qN9ShYG0tDSKiorsGuIIV1RURLNmzRgwYIDTUUwd8eVA0AfADqArMAXYD6T6MVNYWLJkCQDXXnutw0mME1SVJUuWcOrUKSuYYcaXotlKVWcApar6lar+GLjaz7lCWl5eHm+88Qa33norrVu3djqOccDSpUtp1aoVbdu2dTqKqWO+FM1S79ejIjJaRK4AOvoycxEZISI7RWS3iPy6ijY3iMgGEdkqIl/5mDuoTZ06lZycHH7zm984HcUEmNvtJjs7m2uuucZ6mGHKl32afxSR5sATeM7PbAY8XtOHRKQe8HfgFjxXEqWKyAJV3VauTQvgDWCEqh4UkTa1XoMgc+DAAV5++WXuvPNOu6NkhHG73SxYsIBu3brRt29fp+MYP6mxaKrqIu/TU8CNcPaKoJoMBHar6l7vZ2YC44Bt5drcBXysqge9y8r0PXrwUVV+9atfISK89NJLTscxAbZr1y4uueQSLr30UqejGD+q7uT2esBkPNecL1HVLSJyK/AcEAPUdI+GDkD50ZEOAVdVaNMDaCAiK/CcC/pXVX2/VmsQRF5++WU++eQTpk6dykUXXeR0HBMgLpeL5cuXc9NNN9ndRCNAdT3NGcBFwFrgNRE5AAwCfq2q832Yd2Vn8Fa89WV9oD9wE55CvEpEVqvqrh/MSORnwM8AOnXq5MOiA2/p0qU8++yz3HHHHTzxxBNOxzEB4nK5mDt3Lv369bOCGSGqK5oJQF9VdYtINJANXKKqx3yc9yE8RfeMjsCRStpkq2o+kC8iycDlwA+KpqpOB6aD5xa+Pi4/oF577TU6derEjBkz7IqPCFFaWoqqcs0119Cxo0/HRk0YqO5fY4mqugFUtQjYVYuCCZ5zObuLSFfvOJx3AAsqtPkEGCIi9UWkMZ7N95Abq7OsrIyUlBRGjBhB48aNnY5jAqCkpITZs2eTk5NjBTPCVNfT7CUim7zPBbjYOy2Aqmq1hwdV1SUijwJL8Qzw8Y6qbhWRh73vT1PV7SKyBNgEuIG3VXXLea5TwG3atInvvvvOrv6JIJ9//jmDBw+28zAjUHVFs/f5zlxVk4CkCq9NqzA9FZh6vstyUnJyMgBDhgxxOInxt6KiIvbs2cOoUaNsN0yEqm7ADhukw0fJycl07drVjpiHucLCQubMmcPQoUOtYEYwO9x3nlSV5ORk62WGOVUlKyuLm2++mQ4dOjgdxzjIiuZ5Wrp0KdnZ2Vx//fVORzF+kpeXx8yZM+nQoQPt2rVzOo5xmE9FU0RiRKSnv8OEmszMTB544AF69+7NHXfc4XQc4wclJSXMnTuXm2++mXr17IYFxoeiKSJjgA3AEu90PxGpeOpQxFFVHnjgAXJzc5k5c6adahSGTp06RXFxMZMnTyYuLs7pOCZI+NLT/B2e68hPAqjqBqCLvwKFij179pCUlMSLL75ogzOEodzcXObNm4fL5SImJsbpOCaI+FI0Xap6yu9JQsy+ffsAGDx4sMNJjD9s3ryZ8ePHc8EFFzgdxQQZX4aG2yIidwH1RKQ78EvgG//GCn4HDnjOyOrcubPDSUxdys7OZvv27XahgqmSLz3Nx/DcH6gY+BDPEHGP+zFTSDhw4ABRUVF2+kkYyczM5NNPP6Vfv35ORzFBzJeeZk9VfR6wm3eXc+DAATp06ECDBg2cjmLqQH5+PtHR0UyaNInY2Fin45gg5ktP8xUR2SEifxCRPn5PFCI2b95Mz552FlY4OHLkCHPnzqVx48ZWME2NaiyaqnojcAOQBUwXkc0i8oK/gwWz06dPs2nTJjsIFAZcLhfJyckkJiZSv74vG14m0vl0cruqHlPV14CH8Zyz+Vt/hgp2q1evxu12c801vtz1wwSrAwcOcPz4cW6//XY7rcj4zJeT23uLyO9EZAvwOp4j5xE9gODKlSuJiori6qvtTsahat++faxcuZK4uDgbfMPUii/bI/8P+DcwTFUrjrwekVJSUrjsssto1qyZ01HMOXC73RQWFpKYmGgH8kyt+XI3SutOleNyuVizZg333Xef01HMOdi1axd79+5lxIgRTkcxIaq6u1HOUtXJIrKZH94QzaeR28NVTk4OeXl59O593mM0mwDLyMhg48aNTJgwwekoJoRV19P8lffrrYEIEiqKi4sBiI6OdjiJqY39+/fTvn17brvtNhutyJyXKg8EqepR79NfqOqB8g/gF4GJF3yKiooAaNSokcNJjK82bdpEeno69evXt4JpzpsvpxzdUslrI+s6SKg409O0ohkaysrKyM7OZty4cXZfclMnqtun+XM8Pcpu5e5KCdAUWOnvYMHKNs9DR1paGi1atGDo0KFORzFhpLp9mh8Ci4H/Bn5d7vXTqprj11RBzHqaoSE1NZWsrCz69+/vdBQTZqormqqq+0XkkYpviEjLSC2cVjSD36lTp+jWrRsJCQl24rqpczX1NG8F1uE55aj8b58C3fyYK2itWLECEeHiiy92OoqpxNdff43L5eLGG290OooJU9Xd9/xW79eugYsT3FwuFzNmzGD48OF2j/MgdOjQIVwuFzfccIPTUUwY8+Xa82tEJNb7/B4ReUVEOvk/WvBZvHgxhw8f5qGHHnI6iilHVfnmm2+Ii4vjxhtvtE1y41e+nIPxJlAgIpcDTwMHgH/6NVWQmj59Ou3atWP06NFORzFeqsrnn38O2H5mExi+3lhNgXHAX1X1r3hOO4ooGRkZJCUl8eMf/9gGeQgSqorL5aJLly42tqkJGF+K5mkReRa4F/hUROoBEVc1/vWvf+F2u3nwwQedjmLwFMxFixaxf/9+evTo4XQcE0F8KZq347mp2o9V9RjQAZjq11RB6OOPP+aqq66iS5cuTkcxwFdffUXHjh3p3r2701FMhPHldhfHgA+A5iJyK1Ckqu/7PVkQOXjwIGlpaTY6ThBwu93s2LGDIUOGcMUVVzgdx0QgX46eTwbWAonAZGCNiEzyd7BgMn/+fAArmg4rKytj3rx5uFwuG3jDOMaXkdufBwaoaiaAiMQBy4A5/gwWTObNm0d8fLztO3NYdnY28fHxNpapcZQv+zSjzhRMrxM+fi4s5Obm8vXXXzN+/Hino0Qsl8vF3Llzad68uRVM4zhfeppLRGQpnvsEgefAUJL/IgWXxYsXU1ZWxpgxY5yOEpHcbjdz5swhISHBRpYyQcGXewQ9JSK3Adfiuf58uqrO83uyILFw4ULatGnDwIEDnY4ScUpKSigsLGTUqFF2EzsTNKrczBaR7iLyiffWvYnA/6rqf0VSwVRVvvzyS4YNG2YD2AZYcXExs2bN4rvvvrOCaYJKdZXgHWARMBHPSEd/C0iiILJr1y4yMzO5/vrrnY4ScdavX8/1119vA6OYoFPd5nlTVX3L+3yniKwPRKBgkpycDMB1113ncJLIUVBQQGpqqv2jMkGruqIZLSJX8P04mjHlp1U17ItocnIyF154oV11EiD5+fnMmTOH4cOHOx3FmCpVVzSPAq+Umz5WblqBsL7xSmlpKV9++SVDhgyxocYCoKSkBFVl5MiRtGnTxuk4xlSpukGII3ro6+nTp3PkyBHuv/9+p6OEve+++46PP/6YyZMnW8E0Qc8OCVfi9OnTTJkyheuvv55Ro0Y5HSesqSqffvopY8aMoXHjxk7HMaZGfi2aIjJCRHaKyG4R+XU17QaISFmwXNP+yiuvkJWVxcsvv2yb5n6Um5vL/v37uf3222nVqpXTcYzxid+Kpnfczb8DI4F44E4Ria+i3UvAUn9lqa25c+cydOhQO6Hdj06cOMEnn3zCBRdcYOfAmpDiyyhH4r030G+9051ExJdqMhDYrap7VbUEmIln9PeKHgPmApmVvBdwubm5bNmyxW7O5UeqyvHjx7ntttto0aKF03GMqRVf/sW/AQwC7vROn8bTg6xJByCj3PQh72tniUgHYAIwzYf5BcSqVatQVa699lqno4Sl48eP88knnxAfH29X+piQ5EvRvEpVHwGKAFQ1F2jow+cq2xmoFab/AjyjqmXVzkjkZyKSJiJpWVlZPiz63KWkpFC/fn2uuuoqvy4nEuXk5LB06VJuueUWp6MYc858GeWo1LvfUeHseJpuHz53CCh/DVxH4EiFNgnATO/BltbAKBFxqer88o1UdTowHSAhIaFi4a1TKSkp9O/f347k1rHMzExatGjB5MmTbbQiE9J86Wm+BswD2ojI/wVSgD/58LlUoLuIdBWRhsAdwILyDVS1q6p2UdUueAY1/kXFghlIxcXFrF271jbN61hGRgZLly7F7XZbwTQhz5eh4T4QkXXATXg2ucer6nYfPucSkUfxHBWvB7yjqltF5GHv+0GzH/OMtLQ0iouLrWjWIVVl69atTJ482e5LbsJCjUVTRDoBBcDC8q+p6sGaPquqSVQYsLiqYqmq99c0P39LSUkB4JprrnE4SXjYu3cveXl5jBgxwukoxtQZX/Zpfopnf6YA0UBXYCfQx4+5HJGSkkLPnj2Ji4tzOkrI27NnD2vXrmXSpKC4XsGYOuPLLXwvU9W+3q/d8Zx/meL/aIF18OBBli5daiPs1IGCggKaNm1KYmIiDRo0cDqOMXWq1pdieIeEG+CHLI767//+bwCeeOIJh5OEtu3bt7NkyRLatGlD/fq+bMgYE1p82af5f8pNRgFXAv49WTLAMjIymDFjBj/5yU/o1KmT03FCVm5uLtu3b2fcuMou/DImPPjS02xa7tEIzz7OsPqrONPLfPbZZx1OErq2bt1K/fr1mTBhAvXq1XM6jjF+U21P03tSexNVfSpAeQIuMzOTGTNm8OMf/9h6mecoPT2djIwMevfubaNCmbBX3d0o63svb7wygHkCbsaMGZSUlPD44487HSUkud1uRIQxY8bYaEUmIlTX01yLp2BuEJEFwGwg/8ybqvqxn7P5XVlZGdOmTWPo0KH06tXL6TghZ82aNURFRTFgQNgdFzSmSr4c3mwJnMBzT6Az52sqEPJFMykpiYMHD/Lqq686HSXkbNiwgZMnTzJs2DCnoxgTUNUVzTbeI+db+L5YnuHXQTMCZf78+bRs2ZKxY8c6HSWk7N27l/j4eC6//HLbh2kiTnVFsx7QBN+GeAtJaWlpDBgwwM4n9JGqsnz5cqKioujatasVTBORqr2Fr6r+PmBJAqywsJCtW7cyZswYp6OEjNOnTxMdHc3gwYOdjmKMY6o73BnW3YiNGzdSVlZGQkKC01GCnqry2WefUVZWZgXTRLzqiuZNAUvhgNTUVAD69+/vcJLgt2TJElq0aMEFF1zgdBRjHFfl5rmq5gQySCC5XC7+/ve/Ex8fT8eOHZ2OE7RUlby8PAYMGEDr1q2djmNMUIjIIyAzZsxg586dzJ8/3w5mVMHtdrNw4UK6devGZZdd5nQcY4KGqIbWgfCEhARNS0s758/n5eVxySWX0L17d5KTk61oViE9PZ169erRt29fp6MYExAisk5VazzIEXE9zaVLl3L8+HE+/PBDK5iVKCsr45tvvuHaa6+1748xlYi4i4U3btxIVFQUgwYNcjpK0HG5XMydO5e4uDgrmMZUIeJ6mhs3bqRnz57ExMQ4HSWolJWVUVpayoABA+jatavTcYwJWhHX09y8ebMd2KigpKSEjz76iIKCAiuYxtQgoopmcXExBw4csBGNKli0aBGDBg2iVatWTkcxJuhF1Ob53r17cbvddO/e3ekoQaGoqIijR48yduxYu/7eGB9FVE/z22+/BbCiiefa+9mzZ9OgQQMrmMbUQkT9tVjR/N7Bgwe56aabaN++vdNRjAkpEdfTbNmyJS1btnQ6imPy8/P5+OOP6dGjhxVMY85BRBXNXbt2RXQvs7CwkDlz5tiJ68ach4gqmt9++y09evRwOoYjTp8+TVRUFBMmTKBNmzZOxzEmZEVM0czNzeXQoUMRWTRPnjzJ3LlzKSwspFmzZk7HMSakRUzRXLRoEQC33HKLw0kCb82aNYwfP54WLVo4HcWYkBcxR8/nzZtH+/btI+p2sydOnGDfvn0MHz7c6SjGhI2I6GkWFBSwZMkSxo8fT1RURKwyWVlZLFq0iJ49ezodxZiwEhE9zWXLllFYWMiECROcjhIQJSUluN1uJk6cSJMmTZyOY0xYiYhu15YtWwAiYji4o0ePMmvWLOLi4qxgGuMHEVE0MzIyaNmyJbGxsU5H8auioiK++OILJk6cGDG7IYwJtIjYPM/IyOCiiy5yOoZfZWRk0LhxY+666y4rmMb4UUT8dYV70dy/fz/Jyck0adLECqYxfhYRf2HhXDRVlezsbBITE2nUqJHTcYwJe2G/eZ6fn09ubm5YFs1vv/2Ww4cPc8MNNzgdxZiIEfY9zYyMDAA6derkcJK6tW/fPtavX8+1117rdBRjIkrY9zTPFM1w6mkePXqUdu3aMWnSJOrVq+d0HGMiStj3NPfs2QNA586dHU5SNzZv3sw333xDw4YNrWAa44CwL5opKSm0bds2LDbPS0pKOHz4MBMmTLCj5MY4JKw3z1WVr776iuuuuy7kB91dv349bdq0YcSIEU5HMSai+bW7IiIjRGSniOwWkV9X8v7dIrLJ+/hGRC6vy+UfOHCAQ4cOcd1119XlbAMuNTWVI0eO0KFDB6ejGBPx/NbTFJF6wN+BW4BDQKqILFDVbeWa7QOuV9VcERkJTAeuqqsMycnJACFdNAsLC2nfvj0JCQkh31s2Jhz4c/N8ILBbVfcCiMhMYBxwtmiq6jfl2q8GOtZlgI0bNxITE0OfPn3qcrYBk5KSgtvtDumib0y48efmeQcgo9z0Ie9rVfkJsLiyN0TkZyKSJiJpWVlZPgfIy8ujWbNmIXnQZPfu3ZSUlDBkyBCnoxhjyvFnT7OybUmttKHIjXiKZqVnaqvqdDyb7iQkJFQ6j8oUFBSE3MhGqsq6devo168fF198sW2SGxNk/Fk0DwHlzyjvCByp2EhE+gJvAyNV9URdBsjPz6dx48Z1OUu/UlWWLVtGTEwM9euH9YkNxoQsf/5lpgLdRaQrcBi4A7irfAMR6QR8DNyrqrvqOkBBQUHIFE1VxeVyceGFF9K3b1+n4xhjquC3oqmqLhF5FFgK1APeUdWtIvKw9/1pwG+BVsAb3s1Ql6om1FWGUNk8V1WSkpLo06ePFUxjgpxftwFVNQlIqvDatHLPHwQe9Nfy8/Pzadeunb9mX2eWLVtG27Zt6dKli9NRjDE1COsdZ8G+ee52u8+efG9jYRoTGkLvXJxaCOaiWVZWxvz58zl16pQVTGNCSFj3NPPz84N2n+ahQ4fo1asX8fHxTkcxxtRCWPc0CwsLiY6OdjrGD7hcLhYuXEj79u2tYBoTgsK6aBYXFwdV0XS5XMyZM4devXrRoEEDp+MYY85B2G6el5WVUVZWFjT7C0tKSigrK+Omm24iLi7O6TjGmHMUtj3N4uJigKAomsXFxcyePZsTJ05YwTQmxFnRDICVK1cyZMgQOnas00GcjDEOCNvN82AomoWFhWzevJmhQ4c6lsEYU7fCtqdZVFQEOFc0CwoKmD17dljdBdMYE8Y9zc8++wzAkdN6ysrKKCgoYPjw4Vx44YUBX74xxn/CsqepqkybNo2+ffty1VV1dvcMn5w+fZoPPviAZs2aWcE0JgyFZdFcu3Yt6enpPPzwwwEdxPfMpZGjRo2iYcOGAVuuMSZwwnLzfNq0aTRp0oR77rknYMs8efIkRUVF3HnnnTaAsDFhLCx7mp9//jljxoyhadOmAVleTk4O8+fPJzo62gqmMWEuLP/C8/Pzad26dcCWt2/fPiZMmEDz5s0DtkxjjDPCsmgWFRUF5JrzzMxM0tPTGT58uN+XZYwJDmG3ea6qFBUVERMT49flZGdns3jxYq655hq/LscYE1zCrqd55kogf/Y0c3JyaNq0KYmJiUE7yLExxj/Crqd55kogf/U0Dx06xKeffgpgBdOYCBR2RfPECc+t0/1xUMbtdrNu3ToSExODYiAQY0zghd3m+bfffgtA9+7d63S++/fvp7S0lHHjxtXpfI0xoSXsepr+KJp79uxh1apVdO7cuc7maYwJTWHX09y1axdNmzalTZs2dTK/0tJS6tevz6RJk+wWFcaY8Oxpdu/evU6uOd+xYweLFy+mc+fOVjCNMUAYF83zdfz4cbZs2cLo0aPrIJUxJlyEVdEsKSlh//799OjR47zms3PnTpo3b85tt91GvXr16iidMSYchFXR3LdvH263+7x6mhs2bGD79u00bNiQqKiw+vYYY+pAWFWFI0eOAJzzDcxUlcLCQsaOHWsF0xhTqbA6eq6qAOe0Sb127Vqio6MZNGhQXccyxoSRsOpO5efnA9T6SPe6devIycnhsssu80csY0wYCaue5oYNGxAR+vTp4/Nnjh07Rq9evbjyyisDemsMY0xoCqueZlpaGj169KBZs2Y1tlVVvvrqK7Zv305sbKwVTGOMT8KqaK5bt46EhASf2p44cYKoqChuvPFGP6cyxoSTsCmax44d4/DhwzUWTVVl+fLlxMTEMGTIkAClM8aEi7ApmqmpqQDVFk1VZcmSJcTExBAbGxuoaMaYMBI2B4K++eYbGjRoQP/+/St9X1UpLi7m0ksv5aKLLgpwOmNMuAibnubKlSu58sorKx2x3e12s3DhQvbu3WsF0xhzXsKiaJaUlJCamsrgwYMrfX/NmjV07tyZ+Pj4ACczxoSbsNg8T09Pp6io6D+u5ikrK2PDhg1cffXVdkqRMaZOhEVP8+uvvwb4wdFwl8vFxx9/TExMjBVMY0ydCYueZnJyMj169KBt27aA56BPfn4+/fr1q/N7BRljIltY9DRTUlLO9jJLS0v56KOPEBErmMaYOufXoikiI0Rkp4jsFpFfV/K+iMhr3vc3iciV57KcU6dO0a5dO1SVefPmMXDgQJ8upTTGmNry2+a5iNQD/g7cAhwCUkVkgapuK9dsJNDd+7gKeNP7tdZcLhfHjx9n7NixREdHn194Y4ypgj97mgOB3aq6V1VLgJlAxZuGjwPeV4/VQAsRaVebhZw8eRK32823335LaWmpFUxjjF/5s2h2ADLKTR/yvlbbNtU6cOAAAMOGDbMT140xfufPo+eVneej59AGEfkZ8DOATp06/eC9zp07M2fOHK6++upzjGmMMb7zZ0/zEFC+69cROHIObVDV6aqaoKoJcXFxP3ivRYsWTJw4kQ4datVBNcaYc+LPopkKdBeRriLSELgDWFChzQLgPu9R9KuBU6p61I+ZjDHmvPht81xVXSLyKLAUqAe8o6pbReRh7/vTgCRgFLAbKAAe8FceY4ypC369IkhVk/AUxvKvTSv3XIFH/JnBGGPqkpy57W2oEJEs4EAlb7UGsgMcx9/CcZ3A1iuUhOM6QeXr1VlV4yprXF7IFc2qiEiaqvp2g6AQEY7rBLZeoSQc1wnOb73C4tpzY4wJFCuaxhhTC+FUNKc7HcAPwnGdwNYrlITjOsF5rFfY7NM0xphACKeepjHG+F1IFc1Ajc8ZaD6s193e9dkkIt+IyOVO5KyNmtapXLsBIlImIpMCme9c+bJeInKDiGwQka0i8lWgM54LH34Hm4vIQhHZ6F2voL8QRUTeEZFMEdlSxfvnVi9UNSQeeK4q2gN0AxoCG4H4Cm1GAYvxDARyNbDG6dx1tF6DgQu8z0cG+3r5sk7l2n2J5wKISU7nrqOfVQtgG9DJO93G6dx1tF7PAS95n8cBOUBDp7PXsF7XAVcCW6p4/5zqRSj1NAMyPqcDalwvVf1GVXO9k6vxDGwSzHz5WQE8BswFMgMZ7jz4sl53AR+r6kEAVQ2FdfNlvRRoKp67FDbBUzRdgY1ZO6qajCdnVc6pXoRS0QzI+JwOqG3mn+D57xjMalwnEekATACmETp8+Vn1AC4QkRUisk5E7gtYunPny3q9DvTGMwrZZuBXquoOTDy/Oad6EUp3o6yz8TmDjM+ZReRGPEXzWr8mOn++rNNfgGdUtSyEbrHsy3rVB/oDNwExwCoRWa2qu/wd7jz4sl7DgQ3AUOBi4HMR+VpVv/NzNn86p3oRSkWzzsbnDDI+ZRaRvsDbwEhVPRGgbOfKl3VKAGZ6C2ZrYJSIuFR1fkASnhtffwezVTUfyBeRZOByIJiLpi/r9QDwP+rZGbhbRPYBvYC1gYnoF+dWL5zeWVuLnbr1gb1AV77fWd2nQpvR/HDH7lqnc9fRenXCM3zeYKfz1tU6VWj/LqFxIMiXn1Vv4Atv28bAFuBSp7PXwXq9CfzO+/xC4DDQ2unsPqxbF6o+EHRO9SJkepoapuNz+rhevwVaAW94e2YuDeJBFHxcp5Djy3qp6nYRWQJsAtzA26pa6SkvwcLHn9cfgHdFZDOeIvOMqgb16Eci8m/gBqC1iBwCXgQawPnVC7siyBhjaiGUjp4bY4zjrGgaY0wtWNE0xphasKJpjDG1YEXTGGNqwYqm8Yl3JKIN5R5dqmmbVwfLe1dE9nmXtV5EBp3DPN4WkXjv8+cqvPfN+Wb0zufM92WLdxSgFjW07ycio+pi2cYZdsqR8YmI5Klqk7puW8083gUWqeocERkG/FlV+57H/M47U03zFZH3gF2q+n+raX8/kKCqj9Z1FhMY1tM050REmojIF95e4GYR+Y9RjESknYgkl+uJDfG+PkxEVnk/O1tEaipmycAl3s/+H++8tojI497XYkXkU+9Yj1tE5Hbv6ytEJEFE/geI8eb4wPtenvfrR+V7ft4e7kQRqSciU0Uk1TvW4kM+fFtW4R3wQUQGimfs03Tv154i0hD4PXC7N8vt3uzveJeTXtn30QQZpy9zskdoPIAyPAM2bADm4bn0rpn3vdZ4rqo4s+WS5/36BPC893k9oKm3bTIQ6339GeC3lSzvXbyXVgKJwBo8A2FsBmLxDE+2FbgCmAi8Ve6zzb1fV+Dp1Z3NVK7NmYwTgPe8zxviGfUmBvgZ8IL39UZAGtC1kpx55dZvNjDCO90MqO99fjMw1/v8fuD1cp//E3CP93kLPNeoxzr987ZH1Y+QuYzSOK5QVfudmRCRBsCfROQ6PJcLdsBzTfKxcp9JBd7xtp2vqhtE5HogHljpvSS0IZ4eWmWmisgLQBae0Z1uAuapZzAMRORjYAiwBPiziLyEZ5P+61qs12LgNRFpBIwAklW10LtLoK98P6J8c6A7sK/C52NEZAOea5zXAZ+Xa/+eiHTHM3JOgyqWPwwYKyJPeqej8Yw1sL0W62ACyIqmOVd34xnBu7+qlorIfjx/8GeparK3qI4G/ikiU4Fc4HNVvdOHZTylqnPOTIjIzZU1UtVdItIfz3XE/y0in6nq731ZCVUtEpEVeIY+ux3495nFAY+p6tIaZlGoqv1EpDmwCHgEeA3PtdrLVXWC96DZiio+L8BEVd3pS17jPNunac5VcyDTWzBvBDpXbCAinb1t3gJm4Ln1wGrgGhE5s4+ysYj08HGZycB472di8Wxafy0i7YECVf0X8Gfvcioq9fZ4KzMTz2ANQ/AMWoH368/PfEZEeniXWSlVPQX8EnjS+5nmeEYCAs8m+Rmn8eymOGMp8Jh4u90ickVVyzDBwYqmOVcfAAkikoan17mjkjY3ABtEJB3Pfse/qmoWniLybxHZhKeI9vJlgaq6Hs++zrV49nG+rarpwGXAWu9m8vPAHyv5+HRg05kDQRV8hud+MsvUc7sH8Ixdug1YL54bc/2DGrbMvFk2AncAL+Pp9a7Es7/zjOVA/JkDQXh6pA282bZ4p00Qs1OOjDGmFqynaYwxtWBF0xhjasGKpjHG1IIVTWOMqQUrmsYYUwtWNI0xphasaBpjTC1Y0TTGmFr4/2Xa/6ze59h+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(df_scores.fpr, df_scores.tpr, color='black')\n",
    "plt.plot([0, 1], [0, 1], color='black', lw=0.7, linestyle='dashed', alpha=0.5)\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "\n",
    "# plt.savefig('04_roc_curve.svg')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Scikit-Learn for plotting the ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "fpr, tpr, thresholds = roc_curve(y_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA4O0lEQVR4nO3deXxU9b3/8deHhJCEJSCLAmGzgoCAqAEFZVfZlMUQFP21irelaq31VnvbWu9tbbXtrb3WtVXqVpeCAoKACILKIoiEfRUMEAhLIBDClpBkMp/fHzOJIWaZhJk5s3yej0cemeXMOe8Tkg/fs3y/X1FVjDHG+Kae0wGMMSacWNE0xphasKJpjDG1YEXTGGNqwYqmMcbUghVNY4ypBSuaxhhTC1Y0TcCJSKaIFIjIGRHJFpE3RaRRhWX6i8hnInJaRE6KyDwR6V5hmSYi8qyI7PeuK8P7vEVw98hEMyuaJlhuVdVGQG/gKuDXpW+ISD/gE+BDoA3QCdgErBSRS73LxAGfAlcAI4AmQH/gONA3UKFFJDZQ6zbhyYqmCSpVzQYW4Smepf4CvKWqz6nqaVXNVdXHgdXA77zL/ABoD4xX1e2q6lbVo6r6B1VdUNm2ROQKEVksIrkickREHvO+/qaIPFluucEicqDc80wR+aWIbAbOisjjIjKzwrqfE5HnvY+TROQ1ETksIgdF5EkRibmwn5QJVVY0TVCJSDIwEsjwPk/E02KcUcni7wM3eR/fCCxU1TM+bqcxsARYiKf1ehmelqqvJgGjgabA28AoEWniXXcMMBH4t3fZfwEu7zauAm4GfliLbZkwYkXTBMscETkNZAFHgd96X78Iz+/h4Uo+cxgoPV/ZvIplqnILkK2q/6eq57wt2K9q8fnnVTVLVQtUdR+wHhjnfW8okK+qq0XkYjz/CTysqmdV9SjwN+COWmzLhBErmiZYxqlqY2Aw0JVvi+EJwA20ruQzrYFj3sfHq1imKu2A3XVK6pFV4fm/8bQ+Ae7k21ZmB6A+cFhE8kQkD3gFaHUB2zYhzIqmCSpVXQa8CfzV+/ws8CWQVsniE/n2kHoJMFxEGvq4qSzge1W8dxZILPf8ksqiVng+AxjsPb0wnm+LZhZQCLRQ1aberyaqeoWPOU2YsaJpnPAscJOI9PY+/xVwt4g8JCKNRaSZ90JNP+AJ7zJv4ylQs0Skq4jUE5HmIvKYiIyqZBvzgUtE5GERaeBd77Xe9zbiOUd5kYhcAjxcU2BVzQGWAm8Ae1V1h/f1w3iu/P+f95aoeiLyPREZVMufiQkTVjRN0HkL0FvAf3uffwEMB27Dc95yH54LKjeo6jfeZQrxXAz6GlgMnALW4DnM/865SlU9jeci0q1ANvANMMT79tt4bmnKxFPw3vMx+r+9Gf5d4fUfAHHAdjynG2ZSu1MJJoyIDUJsjDG+s5amMcbUghVNY4ypBSuaxhhTC1Y0jTGmFqxoGmNMLYTdCC4tWrTQjh07Oh3DGBNh1q1bd0xVW9a0XMCKpoi8jqf/71FV7VHJ+wI8B4wC8oF7VHV9Tevt2LEja9eu9XdcY0yUE5F9viwXyMPzN/GMe1iVkUBn79cU4B8BzGKMMX4RsKKpqsuB3GoWGYtnDEVV1dVAUxGxXhTGmJDm5DnNtpw/kswB72u1Gf7LGBOh3nnnHbZv3x6QdV977bWMHTu2Tp91smhKJa9V2qdTRKbgOYSnffv2gcxkjAmiV155hfvuuw+AmJjzB7svKSkBoH79+n7bXmm38fvuuy8si+YBPGMelkoGDlW2oKpOBaYCpKSkWGd5Y4Lg8OHDfPLJJwAUFhbywAMPkJSURGys/8rG0aNHAXjkkUeIj48/77169eoxefJkOnXq5Jdt5eXlMWfOHMaOHUuzZs3qvB4ni+Zc4EERmQ5cC5z0DrNljAmQQ4cOsX79elSVRx55hJiYGDw3snzXjh07vvNao0aNGDWqspH46u7aa6/lnnvu8es6K7N69eoLLpgQ2FuOpuEZpbuFd9Kq3+IZ4RpVfRlYgOd2oww8txxNDlQWYyJJYWEhGzZswJcRyo4dO8YTTzxBQkICIsKKFSu+s0xaWmXjP0OPHj249NJL+fGPfwxAXFwcbdu2vbDwDjh+/Dj79u1jxIjqbubxXcCKpqpOquF9BX4SqO0bEwlyc3PJycnhv//7vykuLgZgzpw5tV5PixYt6NmzJ0OGDKFbt25MnjyZmJgYevXq9Z1ziZEkJyeHBQsWMH78eL+tM+x6BBkTrgoKCigoKDjvtYyMDF555ZVKD5GLi4t56623znutV69e9OzZk8TERJ544onvfKYyjRs3pl+/flUehkeq4uJiSkpKSE1NpVGjRn5brxVNY4IgKyur2js/WrZsSVxc3Hmvud1umjdvzpgxYxg5ciSjR48mMTGxijWY8g4fPsynn37KnXfeSb16/r0d3YqmMQH22Wef8dprrwGeix533nnnee937NiRMWPGOBEtIhUUFLBkyRJSU1P9XjDBiqYxtaKqzJkzhxMnTlS7nNvt5v7776dRo0bk5eUBkJiYyJw5c7jkksomvzT+kJWVRcOGDbnrrrsCUjDBiqYxPtu+fTtjxoxh927fp1OvX78+999/P7fffjv9+vX7ziG48Z99+/axYsUK0tLSAlYwwYqmMTUqKipi/fr1jBkzhpycHADS09Np1apVtZ+LjY2lTZs2wYgY9VSVo0ePMnHixID/x2RF05hqTJs27bxzkFOmTOGZZ56hYcOGDqYy5WVkZHDw4EEGDQrOVPNWNI0Bli1bxhtvvPGdw7o33ngDgN69e/PXv/6V6667zgpmCNm7dy/r1q0jNTU1aNu0ommiWmZmJoMGDWL//v0AtGvX7rz3W7duzd13382f/vQnJ+KZamRnZ3PJJZeQmprq1/7wNbGiaaJORkYGH3zwAc899xyHDn07RszixYu58cYbHUxmfLV161Z27tzJ+PHjA3rRpzJWNE3UGTZsWFnLEjzjNt51110OJjK1UVhYSFZWliMFE6xomgjmdrv59NNPOXPmTNlrjz76KIcOHaJx48YcOXKEBg0aOPKHZ+pmw4YNtGrVipEjRzqWwYqmiRj79+9nwYIFvPLKKzRo0IBNmzZx7ty5SpedPn06CQkJQU5oLsTatWvJzs6md+/ejuawomnCmtvt5ptvvmHFihX86Ec/Knu9T58+DBw4kKKiIp544gmSkpIAz8C23bt3j+iRfSLRuXPnuPjii7nmmmscH3jEiqYJWydPnqRTp07ndWn8xS9+wfe//3169uzpYDLjTytXrsTlcgXtPsyaWNE0YUdVcbvdvPfee2UFc/r06XTu3Jmrr77a4XTGn7755hvOnTvH0KFDnY5SxoqmCXnr169n4cKFgKdgPv744+e9v3fvXjp27OhAMhMoqsqGDRvo1asXl112meOH5OVZ0TQh5eDBgyxYsABVpbCwkIceeqjS5Xr16sWECRO49NJLrWBGGFVlyZIlxMfHB/WmdV+FXiITtd544w3uvffe77zeuXNnnn766bLbTETEr9O6mtChqhQXF9OqVSuuvPJKp+NUyoqmcZzb7SY9Pb2sYD700EP88pe/BDxDq7Vs2dLJeCZIVJWPP/6YK664ImQLJljRNCFgwoQJzJ49G4D77ruP5557zuFExglLlizh4osvpkOHDk5HqZYVTeOojIyMsoL58ccfM3jwYGcDmaBzu90cPHiQAQMGEB8f73ScGln/MeOIr776iocffpjOnTsD8PTTTzNixIiw+KMx/lNSUlI2fUi4/NtbS9MElcvl4h//+EfZVfGEhATi4+N55JFHHE5mnJCVlcXll1/OFVdc4XQUn1lL0wTNli1b6NevX1nBHDduHPn5+eTm5obUfXgm8FwuF/Pnzyc5OTmsCiZYS9MEyd/+9jd+/vOflz0/ePCgzZ8TpVwuF7NmzeKqq64KyfswaxJ+iU1YUFXS09OZN28ec+fOZfPmzQC89tpr3H777TZlRJQqKirC7XYzZMiQGiemC1VWNE1A/Otf/2Ly5Mllz0ePHs2oUaMqvXndRIfCwkJmzpzJoEGDSE5OdjpOnVnRNH63adOmsoL5zjvv0K9fPy699FKHUxmnffHFF9xwww1hXTDBiqbxo3379vHMM8/w/PPPAzBgwAAmTZpkI6NHuYKCArZt28bQoUMj4oKf/TYbv3jhhRfo2LFjWcH86U9/yvLly61gRrn8/HxmzJhBmzZtIqJggrU0jZ989tlngGfQjXvuucfZMCYkuN1uzp49y80338wll1zidBy/sWaAuSAnT57kt7/9LXPmzGHQoEFWMA0Ap0+f5t133yUpKSmiCiZYS9PUoKSkhNWrV1NQUFD22vTp01m7di316tVjw4YNZa/bNLgGPL8zs2fPZtSoUcTFxTkdx++saJoqPfXUUzz55JNVzuh46623kpycTNOmTfnLX/4ScS0KU3t5eXkUFRUxadKkiB3z1IqmOU9ubi5nzpzhwQcfZN68eQD07t2bP//5z+fdkH7ZZZdZkTTnyc3NZe7cuYwbNy5iCyZY0TTlfPnll/Tv3/+819asWUOfPn0cSmTCye7duxk/fnzZdMmRyoqmATznoX74wx8CcO+993L99ddz00030a5dO4eTmVCXk5PDhg0buPnmm52OEhRWNA2HDh2ibdu2Zc9feOEFEhMTHUxkwsWxY8dYsGABqampTkcJGrvlyPDoo48CEBsby8mTJ61gGp+cOHGCRo0akZaWRqNGjZyOEzRWNKNcSUkJ06ZNAzwDKjRp0sThRCYcHDx4kPnz5yMiUfefrBXNKJadnU2DBg0A6NGjh3V5ND5xu92sWbOGtLS0st+faGLnNKOUy+WidevWZc8XLVrkYBoTLjIzM3G5XIwfP97pKI6xpkWUKh3XskOHDhQWFtoo6qZGe/bsYdWqVbRv397pKI4KaNEUkREislNEMkTkV5W8nyQi80Rkk4hsE5HJla3H+F9+fj4AW7dujciubsa/iouLERHS0tKi/vclYEVTRGKAl4CRQHdgkoh0r7DYT4DtqnolMBj4PxGJ7n+RIJk7dy5dunSJqquepm527tzJxx9/TKdOnSK6p4+vAtnS7AtkqOoeVS0CpgNjKyyjQGPxDLTXCMgFXAHMFPW++eYbmjRpQnFxsf0BmBplZ2ezefNmRo8e7XSUkBHIotkWyCr3/ID3tfJeBLoBh4AtwM9U1V1xRSIyRUTWisjanJycQOWNeKrKL37xC06fPk3Pnj2ZOnWq05FMCNu1axdNmzYlNTWVmJgYp+OEjEBePa9smGat8Hw4sBEYCnwPWCwiK1T11HkfUp0KTAVISUmpuA5Tg1OnTjFy5EgyMzM5dOgQAKtXr466++uM7zZu3EhmZiaXXXaZ3YpWQSB/GgeA8h2Xk/G0KMubDHygHhnAXqBrADNFFVUlIyODpKQkVq1aRXZ2Nn379mXVqlVWME2VVJX8/HzGjBljBbMSgWxppgOdRaQTcBC4A7izwjL7gWHAChG5GLgc2BPATFFBVTlw4AA9e/bk5MmTZa8XFhYSG2u35pqqpaenEx8f/53Rrsy3AvbfiKq6gAeBRcAO4H1V3SYi94nIfd7F/gD0F5EtwKfAL1X1WKAyRYvhw4fTvn37soL5/vvvU1BQYAXTVGvdunUcO3aMHj16OB0lpIlqeJ0iTElJ0bVr1zodIySVlJSQnJxMdnY2AK+99hp33323ncQ3NTp69CiJiYk0bNgwYmaNrC0RWaeqKTUtZ02PCLJw4cKygrlhwwZ69+7tbCATFpYtW0ZJSQlDhw51OkpYsLO8EWT69OmA57yUFUzji5ycHETECmYtWNGMIPv27aNBgwakpNR4hGGinKqydOlSEhMTGThwoNNxwooVzQjxs5/9jBUrVlgvH1MjVWXRokU0aNDgvMnyjG/snGYE2LlzJ88//zwAc+bMcTaMCWmqSlFREd27d4/60YrqylqaYe6ll16ia1dPf4C//e1vDBs2zOFEJlS53W7mzZvH7t27rWBeAGtphrGNGzfy4IMPAjB16lTuuusuhxOZULZ69Wrat29P9+4VBxsztWFFM4zt3bsXgF//+tf86Ec/cjiNCVUlJSVs2rSJfv36Re09mP5kh+dhqqSkhCVLlgBw++23O5zGhKqSkhJmz55NfHy8FUw/sZZmGHK5XHTu3JnMzEwAkpKSnA1kQpKqcubMGXr16kWXLl2cjhMxrKUZRv75z38yduxY6tevX1YwN23aRMeOHR3NZUJPcXEx77//PvXq1bOC6WfW0gwjU6ZMoXHjxlx55ZWUlJTwxRdfWCvTfIeqMnv2bFJSUmjcuLHTcSKOFc0w8V//9V8A9OrViy+++MLhNCZUFRYWkpeXx5gxY4iPj3c6TkSyohniVJVBgwaxYsUKAN59912HE5lQde7cOWbOnMnAgQOtYAaQFc0Qt2PHjrKCuWbNGjp06OBwIhOqduzYweDBg0lOTnY6SkSzohnCDh06xBVXXAHAO++8Q58+fRxOZEJRfn4+y5cvZ8SIEU5HiQp29TyEPfTQQ2WP7V5MU5mzZ88yc+ZMrrrqKqejRA0rmiFq9erVzJo1i0aNGlFcXGxTVZjvyM/PJzY2ltGjR3PxxRc7HSdqWNEMQapKv379AHjsscesYJrvOHXqFDNmzKCwsJDmzZs7HSeqWNEMMapaduN6QkICv/71r50NZELSp59+yq233kqTJk2cjhJ1rAkTQlwuFxdddBGnT58GPEO9GVNebm4uR44cYdy4cdaX3CHW0gwRbrebkSNHlhXMZ599lnvvvdfhVCaUHDt2jLlz59K2bVsrmA6ylmYIOHHiBN27dy+bSfLo0aO0bNnS4VQmlLjdbk6fPk1qaqp1jXSYFc0QcNFFF5U93rdvnxVMc57s7GyWLVvGxIkTrYUZAqxoOujcuXOsXr267HlBQYF1fzPnOX36NJ988gkTJkywghkirGg6qGPHjhw5cgSAadOmWcE05zl06BBJSUlMmjTJZhkNIXYhyCGnT58uK5jz5s0jNTXV4UQmlOzfv5/PPvuMmJgYK5ghxlqaDvnwww8B+M1vfsMtt9zicBoTajIzM0lLS6NBgwZORzEVWNF0yFtvvQXAHXfc4XASE0p2797NsWPHGDhwoNNRTBV8PjwXkYaBDBItTp06RevWrVm8eDFA2ShGxuzevZv09HSuvvpqp6OYatRYNEWkv4hsB3Z4n18pIn8PeLIIo6o89thjJCUlkZ2dTWxsLIsXL7YrogaAvLw8WrRowYQJE+wcZojzpaX5N2A4cBxAVTcBduxQS5988gl/+tOfAJg4cSJFRUXceOONDqcyoWDbtm0sWbKEJk2a2OAsYcCnw3NVzarwUkkAskSsH/3oR2UDxL799tu899571sI0gGc8zIyMDMaPH2+/E2HCl6KZJSL9ARWROBF5FO+huqnZZ599xquvvgrACy+8wKRJkxxOZELFpk2bKCgoYOzYscTExDgdx/jIl2OB+4DngLbAAeAT4IFAhooUqsqwYcMAz+H5TTfd5HAiEyrWrVvHoUOH6Nmzp9NRTC35UjQvV9W7yr8gItcDKwMTKXLs3LkT8PT8sYJpSrlcLho3bswtt9xih+RhyJfD8xd8fM1UsHTpUgD+8Ic/OBvEhIwvv/ySr776ii5duljBDFNVtjRFpB/QH2gpIj8v91YTwE7A+ODxxx8HYMiQIQ4nMaFg27ZtnDlzxu6aCHPVHZ7HAY28y5QfwO8UMCGQoSKBy+Xi+PHjALRt29bhNMZJqsqOHTu4/PLL6d69u7Uww1yVRVNVlwHLRORNVd0XxEwR4dChQwCkpaU5nMQ4SVX57LPPqF+/Pt27d3c6jvEDXy4E5YvI08AVQNnYZao6NGCpIsA777wDwJgxYxxOYpx07tw5mjRpQp8+fZyOYvzElwtB7wJfA52AJ4BMID2AmSLC7NmzAbjtttscTmKcoKosXLiQkydPWsGMML4Uzeaq+hpQrKrLVPVe4LoA5wpbubm5PPvss6xduxaAxMREhxMZJyxatIjmzZtzySWXOB3F+JkvRbPY+/2wiIwWkauAZF9WLiIjRGSniGSIyK+qWGawiGwUkW0isszH3CHr7bff5j//8z8BePjhh50NY4LO7XZz7Ngxrr/+emthRihfzmk+KSJJwCN47s9sAjxc04dEJAZ4CbgJT0+idBGZq6rbyy3TFPg7MEJV94tIq1rvQYjZvXs3AAcPHqRNmzYOpzHB5Ha7mTt3Lpdeeim9evVyOo4JkBqLpqrO9z48CQyBsh5BNekLZKjqHu9npgNjge3llrkT+EBV93u3ddT36KFHVXnhBc99/82bN3c4jQm2Xbt2cdlll9GjRw+no5gAqvLwXERiRGSSiDwqIj28r90iIquAF31Yd1ug/OhIB7yvldcFaCYiS0VknYj8oJb5Q8rRo56af9VVV9k0BVHE5XKxePFiunTpYgUzClTX0nwNaAesAZ4XkX1AP+BXqjrHh3VXdgevVrL9a4BhQALwpYisVtVd561IZAowBaB9+/Y+bDr40tPT6du3LwDf//73HU5jgsXlcjFr1ix69+5NvXo2T2E0qK5opgC9VNUtIvHAMeAyVc32cd0H8BTdUsnAoUqWOaaqZ4GzIrIcuBI4r2iq6lRgKkBKSkrFwhsSSu/HbNq0Kffff7/DaUwwFBcXo6pcf/31JCf7dG3URIDq/mssUlU3gKqeA3bVomCC517OziLSSUTigDuAuRWW+RAYICKxIpIIXEsYjtU5bdo0srOzadu2LSdOnLD5y6NAUVERM2bMIDc31wpmlKmupdlVRDZ7HwvwPe9zAVRVq708qKouEXkQWIRngI/XVXWbiNznff9lVd0hIguBzYAbeFVVt17gPgVd6SDDr7/+usNJTLAsXryY/v37232YUUhUKz/aFZEO1X3Qqf7oKSkpWnrjeKiIjY2lpKSEqn6WJnKcO3eO3bt328AbEUhE1qlqSk3LVTdghw3S4YN9+/ZRUlJCt27dnI5iAqygoICZM2cydOhQK5hRzKa+u0DZ2Z7TvI888ojDSUwgqSo5OTnceOONtG7d2uk4xkF2j8QFKCgo4LrrPN3wW7Ro4XAaEyhnzpxh+vTptG3b1gqm8a1oikiCiFwe6DDh5NSpU2WDcbRr165sil4TWYqKipg1axY33nijzRhpAB+KpojcCmwEFnqf9xaRircORZ28vDwAUlJS2L59u/UAikAnT56ksLCQiRMn0rJlS6fjmBDhS0vzd3j6kecBqOpGoGOgAoWLdevWAfDjH/+YRo0aOZzG+NuJEyeYPXs2LpeLhIQEp+OYEOJL0XSp6smAJwkzTz75JICNZhOhtmzZwrhx42jWrJnTUUyI8eXq+VYRuROIEZHOwEPAqsDGCn2lFwRK+5ubyHDs2DF27NjBwIEDnY5iQpQvLc2f4pkfqBD4N54h4h4OYKawcc011zgdwfjR0aNH+eijj+jdu7fTUUwIq7JHUNkCIlep6oYg5alRqPQIaty4MV26dCk7t2nC29mzZykpKSEmJoaGDRs6Hcc4wNceQb60NJ8Rka9F5A8icoUfsoW9s2fPcubMGbKysmpe2IS8Q4cOMWvWLBITE61gmhrVWDRVdQgwGMgBporIFhF5PNDBQtnvfvc7AKZMmeJsEHPBXC4Xy5cvJy0tjdhY6yBnalbj4fl5C4v0BP4LuF1V4wKWqhqhcHhe2u/4yJEjtGoV9tMaRa19+/YRGxtLmzZtrC+58d/huYh0E5HfichWPNNcrMLH2SgjkcvlAuCyyy6zghnG9u7dy8qVK2nZsqUVTFMrvhyPvAFMA25W1Yojr0edU6dOAXD99b7MLWdCkdvtpqCggLS0NOrXr+90HBNmfJmN8rpgBAk3V199tdMRTB3s2rWLPXv22FgBps6qLJoi8r6qThSRLZw/IZpPI7dHqh07PLNxFBQUOJzE1FZWVhabNm1i/PjxTkcxYay6lubPvN9vCUaQcFFaLO0G6PCSmZlJmzZtuO2222y0InNBqrwQpKqHvQ8fUNV95b+AB4ITL3TZ/XzhY/PmzWzYsIHY2FgrmOaC+XJz+02VvDbS30GMCYSSkhKOHTvG2LFjbV5y4xfVndO8H0+L8tJys1ICNAZWBjqYMRdq7dq1NG3alKFDhzodxUSQ6s5p/hv4GPgT8Ktyr59W1dyApjLmAqWnp5OTk2ODqhi/q65oqqpmishPKr4hIhdFa+HcuXOn0xFMDU6ePMmll15KSkqK3bhu/K6mluYtwDo8txyV/+1T4NIA5gpZv/nNbwBo3769w0lMZVasWIHL5WLIkCFORzERqrp5z2/xfu8UvDih7bHHHuPkSc8g9snJUduTNGQdOHAAl8vF4MGDnY5iIpgvfc+vF5GG3sf/T0SeEZGobGZlZmYCnsFq7Ups6FBVVq1aRcuWLRkyZIgdkpuA8uUv/x9AvohciWeEo33A2wFNFaKmTZtGcnKyzUwYQlSVxYsXA9iMoCYofJ1YTYGxwHOq+hye246iypkzZwDIzs52OIkppaq4XC46duxI//79nY5jooQvRfO0iPwa+D7wkYjEAFE3NMyLL74IwFNPPeVwEgOegjl//nwyMzPp0qWL03FMFPFljqBLgDuBdFVd4T2fOVhV3wpGwIqcGoS49DzZiRMnaNq0adC3b863dOlSkpKSuOqqq5yOYiKEr4MQ+zI0XLaIvAv0EZFbgDVOFUynlM4FlJycbAXTYW63m127djFgwADrR24c4cvV84nAGiANmAh8JSITAh0slPzqV54OUb///e8dThLdSkpKmD17Ni6XywqmcYwvh+ebgJtU9aj3eUtgiapeGYR83xHsw3O32132B3r27FkSExODtm1zviNHjpCbm0u3bt2cjmIikD+n8K1XWjC9jvv4uYjw4YcfAtC/f38rmA5xuVzMmjWLpKQkK5jGcb7MEbRQRBbhmScI4HZgQeAihZbnn3/+vO8muNxuNzNnziQlJYX4+Hin4xjj2xS+InIbcAOe/ufLVXV2oINVJdiH56VXze08WvAVFRVRUFCAiNCkSROn45gId8GH5yLSWUQ+9E7dmwb8n6r+p5MF0yl9+/a1ghlkhYWFvP/++5w6dcoKpgkp1Z2bfB2YD6TiGenohaAkCiFHjhwBoFevqJxDzlHr169n0KBBtGvXzukoxpynunOajVX1n97HO0VkfTAChZLly5cD0KNHD4eTRI/8/HzS09MZNGiQ01GMqVR1RTNeRK7i23E0E8o/V9WIL6KlY2fa2IzBcfbsWWbOnMnw4cOdjmJMlaq8ECQin1fzOVVVRyZeCeaFoMTERAoKCnC73TbcWIAVFRVRVFREfn4+rVq1cjqOiUIX3I1SVaO6ebVmzRoKCgp44IEHrGAG2KlTp/jggw+YOHGiFUwT8qLmJvXaOHHiBH/84x8BuPHGGx1OE9lUlY8++ohbb73VOg+YsODTfZp1XrnICOA5IAZ4VVX/XMVyfYDVwO2qOrO6dQbj8LxevXqoKjExMbhcroBuK5qdOHGCvLw8OnToYCPhG8f5sxtlXQPEAC8BI4HuwCQR6V7Fcv8LLApUltpIT0+n9D+SjIwMh9NEruPHj/Phhx/SrFkzK5gmrPgyypF45wb6H+/z9iLS14d19wUyVHWPqhYB0/GM/l7RT4FZwNFK3gu6efPmAfDRRx/RsWNHZ8NEKFXlyJEj3HbbbTbUngk7vvwX/3egHzDJ+/w0nhZkTdoCWeWeH/C+VkZE2gLjgZd9WF9QlN6babcZBcaRI0f48MMP6d69u/X0MWHJl6J5rar+BDgHoKongDgfPlfZJeeKJ1CfBX6pqiXVrkhkioisFZG1OTk5Pmy6blwuF8uWLQMgISEhYNuJVrm5uSxatIibbrrJ6SjG1JkvoxwVe887KpSNp+n24XMHgPJ94JKBQxWWSQGme2/paQGMEhGXqs4pv5CqTgWmgudCkA/brpPPP/fcmmqH5f539OhRmjZtysSJE220IhPWfGlpPg/MBlqJyFPAF8AfffhcOtBZRDqJSBxwBzC3/AKq2klVO6pqR2Am8EDFghlMa9asAWD69OlORYhIWVlZLFq0CLfbbQXThD1f5gh6V0TWAcPwHHKPU9UdPnzOJSIP4rkqHgO8rqrbROQ+7/shcx6zVOl9gl27dnU4SeRQVbZt28bEiRNtXnITEWosmt7ZJ/OBeeVfU9X9NX1WVRdQYcDiqoqlqt5T0/pMeNmzZw9nzpxhxIgRTkcxxm98Oaf5EZ7zmQLEA52AncAVAczliDlz5jgdIWLs3r2bNWvWMGFCVM3BZ6KAL4fnPcs/F5GrgR8HLJFD9u7dW3a7UcOGDR1OE97y8/Np3LgxaWlpxMb68v+yMeGj1l0xvEPC9QlAFkdNnjwZ8MwFZH/odbdjxw4WLlxIq1at7OdoIpIv5zR/Xu5pPeBqIHA3Szrgyy+/LLs/8z/+4z8cThO+Tpw4wY4dOxg7trKOX8ZEBl9amo3LfTXAc44zov4q+vfvD8Ddd99tI+3U0bZt24iNjWX8+PE2n5KJaNW2NL03tTdS1V8EKU/Q7dmzB4CRI0fyxhtvOJwmPG3YsIGsrCy6detmY4+aiFfdbJSx3u6NVwcxT9C98IJnvrgf/OAH9gdfB6Wj2t966602WpGJCtW1NNfgKZgbRWQuMAM4W/qmqn4Q4GwBl5OTw7PPPgvANddc42yYMPTVV19Rr149+vSJuOuCxlTJl8ubFwHHgaF8e7+mAmFfNEv/2B9++GE6d+7scJrwsnHjRvLy8rj55pudjmJMUFU3sdoB4Bm+LZLlj11VVZ8JfLzv8tfI7fn5+WX3Y9rEabWzZ88ekpOTqV+/vv3cTMS44InV8PQXb4RvQ7yFndI5gB555BH7w/eRqvL5559Tr149OnXqZD83E5WqK5qHVfX3QUvikD//udJpi0wlTp8+TXx8fNktWsZEo+oud0Z8MyImJsZ6rfhAVfnkk08oKSmxgmmiXnVFc1jQUgTZxo0beeqppygpqXbAeOO1cOFCmjZtSrNmzZyOYozjqmxmqWpuMIME09y5nrGQ77jjDoeThDZV5cyZM/Tp04cWLVo4HceYkBDVdyO/++67TkcIWW63m7lz55KZmWkF05hyoq5ofv311/z2t791OkbI27RpE506daJnz541L2xMFIm6qyAffvghAPfff791+6tESUkJq1at4oYbbrBbioypRNRVjeeeew7w9AIy53O5XMyaNYuWLVtawTSmClHX0oyPj2f48OF06dLF6SghpaSkhOLiYvr06UOnTp2cjmNMyIq6lma9evXswkYFRUVFvPfee+Tn51vBNKYGUVc0zXfNnz+ffv360bx5c6ejGBPyou7w3Hzr3LlzHD58mDFjxljPKGN8FHUtzd27dzsdISQUFBQwY8YM6tevbwXTmFqIqr+WnTt3AnDkyBGHkzhv//79DBs2jDZt2jgdxZiwElUtzfJTW0Srs2fP8sEHH9ClSxcrmMbUQVQVzby8PIConWK2oKCAmTNn2o3rxlyAKkduD1V1HbldVcsGzy2dgTKanD59mri4OAoLC2nSpInTcYwJOb6O3B41Lc2tW7cCntZWtMnLy2PWrFkUFBRYwTTmAkVN0Tx9+jQAL730ksNJgu+rr75i3LhxNG3a1OkoxoS9qLp6DtCoUSOnIwTN8ePH2bt3L8OHD3c6ijERIypamm63m4EDBwJEzQWQnJwc5s+fz+WXX+50FGMiSlS0NGfMmFE2tUW/fv0cThN4RUVFuN1uUlNTo6plbUwwREVLMyMjA/AMrBvpReTw4cO8//77tGzZMuL31RgnREXRfPzxxwFo166dw0kC69y5c3z66aekpqbaAMvGBEjEH57v378f8FwAiuTZFLOyskhMTOTOO++0gmlMAEX8X1dpL6DSLpSRKDMzk+XLl9OoUSMrmMYEWMS3NDdt2gRAw4YNHU4SGKrKsWPHSEtLIy4uzuk4xkS8iC+apVfNu3fv7nAS//vmm284ePAggwcPdjqKMVEjao7lIq2luXfvXtavX88NN9zgdBRjokrEtzQj0eHDh2ndujUTJkwgJibG6TjGRJWIb2m6XC6nI/jVli1bWLVqFXFxcVYwjXFAxBfNmTNnApCQkOBwkgtXVFTEwYMHGT9+vF0lN8YhEX14/vLLL7No0SIALr74YofTXJj169fTqlUrRowY4XQUY6JaQJsrIjJCRHaKSIaI/KqS9+8Skc3er1UicqU/t186J9CCBQv8udqgS09P59ChQ7Rt29bpKMZEvYC1NEUkBngJuAk4AKSLyFxV3V5usb3AIFU9ISIjganAtf7KEBMTQ8OGDRk5cqS/Vhl0BQUFtGnThpSUlKgZocmYUBbIw/O+QIaq7gEQkenAWKCsaKrqqnLLrwaS/RkgKyuLoqIif64yqL744ovzhrUzxjgvkIfnbYGscs8PeF+ryn8AH1f2hohMEZG1IrI2JyfH5wArV66kuLjY5+VDSUZGBkVFRQwYMMDpKMaYcgLZ0qzsWLLSWdxEZAieolnpndqqOhXPoTspKSk+zwR38OBBkpP92ngNOFVl3bp19O7dm+9973t2SG5MiAlk0TwAlB+LLRk4VHEhEekFvAqMVNXj/gyQmJjIkCFD/LnKgFJVlixZQkJCArGxEX1jgzFhK5B/melAZxHpBBwE7gDuLL+AiLQHPgC+r6q7/B2gfv36XHTRRf5ebUCoKi6Xi4svvphevXo5HccYU4WAFU1VdYnIg8AiIAZ4XVW3ich93vdfBv4HaA783XsY6vJl3uFIo6osWLCAK664wgqmMSEuoMeAqroAWFDhtZfLPf4h8MNAZggHS5Ys4ZJLLqFjx45ORzHG1MBOnDnI7XZz4MABBg4cSIMGDZyOY4zxgXVgdkhJSQlz5szh5MmTVjCNCSPW0nTIgQMH6Nq1a0QOjmxMJLOWZpC5XC7mzZtHmzZtrGAaE4asaAaRy+Vi5syZdO3alfr16zsdxxhTB3Z4HiRFRUWUlJQwbNgwWrZs6XQcY0wdWUszCAoLC5kxYwbHjx+3gmlMmIvYonn27FlOnjyJ2+12OgorV65kwIABYdcP3hjzXRF7eH7kyBHA0//cKQUFBWzZsoWhQ4c6lsEY418R29Is5dQV6vz8fGbMmEG7du1qXtgYEzYisqWpqqxZs8ax7ZeUlJCfn8/w4cPDfm4iY8z5IrKl+eKLLzJp0iSAoF94OX36NO+++y5NmjSxgmlMBIrIovnwww8DMHfu3KDO3ljaNXLUqFHExcUFbbvGmOCJyMPzVq1akZiYyK233hq0bebl5XHu3DkmTZpkAwgbE8EirqVZVFREdnY2gwYNCto2c3NzmTNnDvHx8VYwjYlwEfcXnpeXBxDUkYP27t3L+PHjSUpKCto2jTHOiLiiWapnz54B38bRo0fZsGEDw4cPD/i2jDGhIeIOz4Pl2LFjfPzxx1x//fVORzHGBFHEtjQDKTc3l8aNG5OWluZojyNjTPBZS7OWDhw4wEcffQQ420XTGOMMK5q14Ha7WbduHWlpaTZFhTFRKuIOz0+ePBmQ9WZmZlJcXMzYsWMDsn5jTHiIuJbmjBkzAGjevLnf1rl7926+/PJLOnTo4Ld1GmPCU8S1NFeuXAnA+PHj/bK+4uJiYmNjmTBhgk1RYYyJvJZmfn4+gF8K3Ndff83HH39Mhw4drGAaY4AIK5rnzp1j6dKldOjQARG5oHUdOXKErVu3Mnr0aD+lM8ZEgogqmidOnAC44B46O3fuJCkpidtuu42YmBh/RDPGRIiIKprbt28HoGvXrnVex8aNG9mxYwdxcXHUqxdRPx5jjB9EVFUoPSS/5ppr6vR5VaWgoIAxY8ZYwTTGVCrirp7X1Zo1a4iPj6dfv35ORzHGhLCIak4988wzdfrcunXryM3NDcrISMaY8BZRLc1vvvkGqN2wcNnZ2XTt2pWrr776gq+4G2MiX8S0NPPz89m1axcTJ06kWbNmNS6vqixbtowdO3bQsGFDK5jGGJ9ETEtzw4YNADRs2NCn5Y8fP069evWCOi2GMSb8RUxLs1Tp1L1VUVU+//xzEhISGDBgQJBSGWMiRcQUzVOnTtW4jKqycOFCEhISfG6RGmNMeRFzeL506VKg6sNzVaWwsJAePXrQrl27ICYzxkSSiGhpqip/+ctfAOjdu/d33ne73cybN489e/ZYwTTGXJCIKJputxuAVq1aVToFxVdffUWHDh3o3r17sKMZYyJMRBye7969G4DJkyef93pJSQkbN27kuuuus1uKjDF+EREtzbS0NACuvfbastdcLhcffPABCQkJVjCNMX4TES3NzZs3A5SNfamqnD17lt69e9O5c2cnoxljIkxEtDQBUlNTiYuLo7i4mPfeew8RsYJpjPG7gBZNERkhIjtFJENEflXJ+yIiz3vf3ywiV9dlO/Xq1aNbt26oKrNnz6Zv3740adLkwnfAGGMqCNjhuYjEAC8BNwEHgHQRmauq28stNhLo7P26FviH93utuN1uXC4XR44cYcyYMcTHx1/4DhhjTCUC2dLsC2So6h5VLQKmAxUnDR8LvKUeq4GmItK6NhspvXK+bt06iouLrWAaYwIqkEWzLZBV7vkB72u1XaZaZ86cAWDcuHF247oxJuACefW8svt8tA7LICJTgCkA7du3P++9Dh06MHPmTK677ro6xjTGGN8FsqV5ACjf9EsGDtVhGVR1qqqmqGpKy5Ytz3uvadOmpKam0rZtrRqoxhhTJ4EsmulAZxHpJCJxwB3A3ArLzAV+4L2Kfh1wUlUPBzCTMcZckIAdnquqS0QeBBYBMcDrqrpNRO7zvv8ysAAYBWQA+cDkqtZnjDGhIKA9glR1AZ7CWP61l8s9VuAngcxgjDH+JJ66FT5EJAfYV8lbLYBjQY4TaJG4T2D7FU4icZ+g8v3qoKotK1u4vLArmlURkbWqmuJ0Dn+KxH0C269wEon7BBe2XxHT99wYY4LBiqYxxtRCJBXNqU4HCIBI3Cew/QonkbhPcAH7FTHnNI0xJhgiqaVpjDEBF1ZFM1jjcwabD/t1l3d/NovIKhG50omctVHTPpVbro+IlIjIhGDmqytf9ktEBovIRhHZJiLLgp2xLnz4HUwSkXkissm7XyHfEUVEXheRoyKytYr361YvVDUsvvD0KtoNXArEAZuA7hWWGQV8jGcgkOuAr5zO7af96g808z4eGer75cs+lVvuMzwdICY4ndtP/1ZNge1Ae+/zVk7n9tN+PQb8r/dxSyAXiHM6ew37NRC4Gthaxft1qhfh1NIMyvicDqhxv1R1laqe8D5djWdgk1Dmy78VwE+BWcDRYIa7AL7s153AB6q6H0BVw2HffNkvBRqLZ5bCRniKpiu4MWtHVZfjyVmVOtWLcCqaQRmf0wG1zfwfeP53DGU17pOItAXGAy8TPnz5t+oCNBORpSKyTkR+ELR0defLfr0IdMMzCtkW4Geq6g5OvICpU70Ip9ko/TY+Z4jxObOIDMFTNG8IaKIL58s+PQv8UlVLwmiKZV/2Kxa4BhgGJABfishqVd0V6HAXwJf9Gg5sBIYC3wMWi8gKVT0V4GyBVKd6EU5F02/jc4YYnzKLSC/gVWCkqh4PUra68mWfUoDp3oLZAhglIi5VnROUhHXj6+/gMVU9C5wVkeXAlUAoF01f9msy8Gf1nAzMEJG9QFdgTXAiBkTd6oXTJ2trcVI3FtgDdOLbk9VXVFhmNOef2F3jdG4/7Vd7PMPn9Xc6r7/2qcLybxIeF4J8+bfqBnzqXTYR2Ar0cDq7H/brH8DvvI8vBg4CLZzO7sO+daTqC0F1qhdh09LUCB2f08f9+h+gOfB3b8vMpSE8iIKP+xR2fNkvVd0hIguBzYAbeFVVK73lJVT4+O/1B+BNEdmCp8j8UlVDevQjEZkGDAZaiMgB4LdAfbiwemE9gowxphbC6eq5McY4zoqmMcbUghVNY4ypBSuaxhhTC1Y0jTGmFqxoGp94RyLaWO6rYzXLnvHD9t4Ukb3eba0XkX51WMerItLd+/ixCu+tutCM3vWU/ly2ekcBalrD8r1FZJQ/tm2cYbccGZ+IyBlVbeTvZatZx5vAfFWdKSI3A39V1V4XsL4LzlTTekXkX8AuVX2qmuXvAVJU9UF/ZzHBYS1NUyci0khEPvW2AreIyHdGMRKR1iKyvFxLbID39ZtF5EvvZ2eISE3FbDlwmfezP/eua6uIPOx9raGIfOQd63GriNzufX2piKSIyJ+BBG+Od73vnfF+f698y8/bwk0VkRgReVpE0r1jLf7Yhx/Ll3gHfBCRvuIZ+3SD9/vlIhIH/B643Zvldm/2173b2VDZz9GEGKe7OdlXeHwBJXgGbNgIzMbT9a6J970WeHpVlB65nPF+fwT4jfdxDNDYu+xyoKH39V8C/1PJ9t7E27USSAO+wjMQxhagIZ7hybYBVwGpwD/LfTbJ+30pnlZdWaZyy5RmHA/8y/s4Ds+oNwnAFOBx7+sNgLVAp0pynim3fzOAEd7nTYBY7+MbgVnex/cAL5b7/B+B/+d93BRPH/WGTv9721fVX2HTjdI4rkBVe5c+EZH6wB9FZCCe7oJt8fRJzi73mXTgde+yc1R1o4gMAroDK71dQuPwtNAq87SIPA7k4BndaRgwWz2DYSAiHwADgIXAX0Xkf/Ec0q+oxX59DDwvIg2AEcByVS3wnhLoJd+OKJ8EdAb2Vvh8gohsxNPHeR2wuNzy/xKRznhGzqlfxfZvBsaIyKPe5/F4xhrYUYt9MEFkRdPU1V14RvC+RlWLRSQTzx98GVVd7i2qo4G3ReRp4ASwWFUn+bCNX6jqzNInInJjZQup6i4RuQZPP+I/icgnqvp7X3ZCVc+JyFI8Q5/dDkwr3RzwU1VdVMMqClS1t4gkAfOBnwDP4+mr/bmqjvdeNFtaxecFSFXVnb7kNc6zc5qmrpKAo96COQToUHEBEengXeafwGt4ph5YDVwvIqXnKBNFpIuP21wOjPN+piGeQ+sVItIGyFfVd4C/erdTUbG3xVuZ6XgGaxiAZ9AKvN/vL/2MiHTxbrNSqnoSeAh41PuZJDwjAYHnkLzUaTynKUotAn4q3ma3iFxV1TZMaLCiaerqXSBFRNbiaXV+Xckyg4GNIrIBz3nH51Q1B08RmSYim/EU0a6+bFBV1+M517kGzznOV1V1A9ATWOM9TP4N8GQlH58KbC69EFTBJ3jmk1minukewDN26XZgvXgm5nqFGo7MvFk2AXcAf8HT6l2J53xnqc+B7qUXgvC0SOt7s231PjchzG45MsaYWrCWpjHG1IIVTWOMqQUrmsYYUwtWNI0xphasaBpjTC1Y0TTGmFqwommMMbVgRdMYY2rh/wM9NjmUdnijAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(fpr, tpr, color='black')\n",
    "plt.plot([0, 1], [0, 1], color='black', lw=0.7, linestyle='dashed', alpha=0.5)\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AUC: Area under the ROC curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores_small = tpr_fpr_dataframe(y_val, y_pred_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8359150837721111"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc(df_scores.fpr, df_scores.tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8108718050089552"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc(df_scores_small.fpr, df_scores_small.tpr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing multiple models with ROC curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAFNCAYAAACE8D3EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABHyUlEQVR4nO3dd3gVZfbA8e8hCYSEANI7AekoNaKgIr1JixAQ3bXtisrqWndV1N/iWld3XdR1RcQui0ovKlW6lNCrICWQAIEkkFBSb+77++PeXBNIuQm5/XyeJ0/uzLwzc4aQk3dm3iLGGJRSSjmngqcDUEopX6JJUymlSkGTplJKlYImTaWUKgVNmkopVQqaNJVSqhQ0aSqlVClo0lQuJyJxIpIhIhdFJFFEPheRKpeV6SEiP4nIBRFJE5GFItLusjJVRWSyiBy3H+uQfbmWe69IBTJNmspdhhljqgCdgM7A83kbRKQ7sBSYDzQAmgE7gfUi0txepiKwAmgPDAKqAj2AFKCbq4IWkWBXHVv5Jk2ayq2MMYnAEmzJM89bwJfGmHeNMReMMWeNMS8CG4FJ9jL3AE2AaGPMPmOM1RhzxhjzijHmh8LOJSLtRWSZiJwVkdMiMtG+/nMReTVfuV4ikpBvOU5EnhWRXcAlEXlRRGZddux3ReQ9++dqIvKJiJwSkRMi8qqIBF3dv5TyVpo0lVuJSCNgMHDIvhyGrcY4s5Di3wH97Z/7AYuNMRedPE8EsBxYjK322gJbTdVZ44DbgerAV8AQEalqP3YQMAb4n73sF4DFfo7OwADgj6U4l/IhmjSVu8wTkQtAPHAG+Jt9fQ1s/w9PFbLPKSDveWXNIsoUZSiQaIz5lzEm016D3VSK/d8zxsQbYzKMMceAbcBI+7Y+QLoxZqOI1MX2R+AJY8wlY8wZ4N/AnaU4l/IhmjSVu4w0xkQAvYA2/JYMzwFWoH4h+9QHku2fU4ooU5TGwOEyRWoTf9ny/7DVPgHu4rdaZlMgBDglIqkikgp8BNS5inMrL6ZJU7mVMWY18DnwT/vyJWADEFNI8TH8dku9HBgoIuFOnioeuLaIbZeAsHzL9QoL9bLlmUAv++OFaH5LmvFAFlDLGFPd/lXVGNPeyTiVj9GkqTxhMtBfRDrZl58D7hWRP4tIhIhcY39R0x142V7mK2wJaraItBGRCiJSU0QmisiQQs6xCKgnIk+ISCX7cW+0b9uB7RllDRGpBzxRUsDGmCRgFfAZcNQYs9++/hS2N///sjeJqiAi14rIbaX8N1E+QpOmcjt7AvoSeMm+vA4YCNyB7bnlMWwvVG4xxvxqL5OF7WXQL8Ay4DywGdtt/hXPKo0xF7C9RBoGJAK/Ar3tm7/C1qQpDlvC+9bJ0P9nj+F/l62/B6gI7MP2uGEWpXuUoHyI6CDESinlPK1pKqVUKWjSVEqpUtCkqZRSpaBJUymlSkGTplJKlYLPjeBSq1YtExkZ6ekwlFJ+ZuvWrcnGmNollXNZ0hSRT7H1/z1jjLmukO0CvAsMAdKB+4wx20o6bmRkJFu2bCnvcJVSAU5EjjlTzpW3559jG/ewKIOBlvav8cCHLoxFKaXKhcuSpjFmDXC2mCIjsI2haIwxG4HqIqK9KJRSXs2TzzQbUnAkmQT7utIM/6WU8lNff/01+/btw2q1smbNmiu2N27cmGuvvZacnBzWr19/xfZmzZrRtGlTMjMz2bhxo2N9WFgYjz32GCNGjChTXJ5MmlLIukL7dIrIeGy38DRp0sSVMSml3Oijjz7i4YcfBsD2msMmf/fu4OBgLBbLFftu2rSJoKAgjDGl3t62bVufTJoJ2MY8zNMIOFlYQWPMVGAqQFRUlHaWV8oNTp06xdKlSwHIyspiwoQJVKtWjeDgotNGdnY2aWlpBdZVrVqVSpUqkZWVxfnz5wtsy0uOTz/9NFOnTuXChQsFtu/YsYOOHTuWx+WQmprKvHnzGDFiBNdcc02Zj+PJpLkAeFREvgFuBNLsw2wppVzk5MmTLFy4kBMnTjBt2jSCgn6byqhKlSqICFlZWaSkpFyR/ABCQ0PJyMi4Yn3nzp1p1aoVGzduZMeOHURGRhIWZhuytH379tSsWZOUlBT27t17xb6///3vee6555g0aVKBGmblypWLTdCltXHjxqtOmODaJkczsI3SXcs+adXfsI1wjTFmCvADtuZGh7A1ObrfVbEo5U+ysrLYvn07xhiMMRw5coT09HQAmjdvTnh4OOfOnSMhIYH4+HjeeecdgoODqV69OsePHy/yuCNGjKBixYrs2rXLkTBbt27Ntddey7Bhwxg2bBghISE8+uijV+z7wAMPMGjQII4fP86UKVN4/fXXS31dVapUKblQGaSkpHDs2DEGDSquMY/zfG5ouKioKKPtNFWgOHv2LElJSbz00kvk5OQAMG/evFIfR0SIjIwkMjKSJk2a0KdPH4KCgmjRooWjttmpUyeCg4OJj4/n9OnTNGnShDp1fHvWjqSkJH744Qeio6OpWrVqsWVFZKsxJqqkY2rSVMpNMjIyrri1PXToEB999FGBlyB5cnJy+PLLLwus69ChA8YYwsLCePnllx21pwkTJtCgQQNat25NREQEKSkpHD16FIBatWrx4IMPUqFCYPWazsnJISUlhSpVqjhVi3U2afpcN0qlfFF8fHyxLT9q165NxYoVC6yzWq3UrFmT4cOHM3jwYDIzMx3PBIOCgujTpw9/+MMfWLp0Ke+//37AJcXinDp1ihUrVnDXXXeV+7+LJk2lXOynn37ik08+AeDGG2/krrvuKrA9MjKSqKgoZsyYgdVqLbDtkUceISgoyPFSBaBSpUqEhITw2muvMW3aNNdfgI/JyMhg+fLljBo1yiV/SPT2XKlSMMYwb948zp07V2w5q9XKI488QpUqVUhNTQVsb55ff/11srOzee6556hevTrDhg3jyy+/ZMmSJYW+qDhx4gS1a9fm1VdfRUS46667aNWqlSsuzS/Ex8cTHh5O9erVS50w9fZcqXK2b98+hg8fzuHDzk+nbrVaGTRoEA888ABWq5U777zTsa1FixbcfPPNAAwcOJCPPvqIu+++u8D+lStXpkKFCrz88suo4h07doy1a9cSExPj0kcVWtNUqgTZ2dls27aN4cOHk5SUBEBsbGyRb5aHDh3K7t27C6ybPn06I0eOJDk5GbAlw9q1SxyFTDnJGMOWLVvo2LHjFc+GnaU1TaXKwYwZMwo8gxw/fjzvvPMO4eHh/PLLLyQnJ/P6668TEhICwJw5c5g8eTJ9+/Zl69atjv3yGntrN+Dyd+jQIU6cOMFtt7lnqnlNmkoBq1ev5rPPPrvitu6zzz4DbG0Y33jjDdq2bevopdK2bVtHucqVK9O6dWuMMXTv3p2kpCRq1arlvgsIUEePHmXr1q2MGjXKbefUpKkCWlxcHLfddpujp0zjxo0dXflEhHr16tG+fXs6d+7M4MGDAYiOjuazzz5jxowZgK1v9eDBgx1tLYODg6lcubIHriawJCYmUq9ePUaNGlWu3S1LoklTBZxDhw4xZ84c3n33XU6e/G2MmGXLlhEREUHPnj1p3rw5+/fv5/z581SrVo0VK1YgItx0002MGzeOatWqFXipo9xrz549HDhwgOjoaLe3T9WkqQJO3759C/TB/vrrr1m2bBljxoxxNCXKzs4GbLXIzz77jKFDh+rttpfIysoiPj7eIwkTNGkqP2a1WlmxYgUXL150rHvmmWc4ceKEY/n111/n7rvvJjMzk6pVq2KMYdCgQQwZMsRR5r777nNn2KoY27dvp06dOo5HJZ6gSVP5jePHj/P999/z4osvkp2dzaVLlyiuSd0jjzzCn/70JwD+8Ic/uCtMVUZbtmwhMTGRTp06eTQOTZrKp1mtVn799VfWrl3Lgw8+6FhfqVIlatSogdVqpUWLFrzxxhvUqlWLpUuXsmzZMlq1asV//vMfD0auSiMzM5O6devStWvXQgc3cSdt3K58VlpaGpGRkY5uigCPP/447du3p1WrVm5rt6dca/369VgsFpf/PLVxu/JbxhisVisffvhhgYRZt25dEhMTmTx5ssdiU+Xr119/JTMzkz59+ng6FAdNmsrrbdu2jcWLFwOQnp7Oa6+9VmD7iBEj+Oqrr4iIiPBEeMoFjDFs376dDh060KJFC4/fkuenSVN5lRMnTvDDDz9gjCErK4s///nPhZarXLkyzz//PM2bN79ikAvl24wxLF++nNDQULc2WneW90WkAtZnn33GAw88cMX6li1bcuLECcc8OE899RRvv/22Drrrh4wx5OTkUKdOnXKbhbK8adJUHme1WomNjXUkzNq1aztGE4qJieG7777j1KlTGGOIiIjQ23A/ZYzhxx9/pH379l6bMEGTpvICo0ePZu7cuQDUr1+fU6dsMzl/9dVXDB8+3LFe+bfly5dTt25dmjZt6ulQiqVJU3nUoUOHHAlz1qxZNGrUiOzsbG688cYyj4uofIvVauXEiRPceuuthIaGejqcEmnSVB6xadMmZsyYwbvvvutY179//xKnWVX+JTc3l/nz59OiRQsaN27s6XCcoklTuZXFYuHDDz+84q34Cy+8oAkzAMXHx9O6dWvat2/v6VCcpklTuc3u3bt54IEHyOvR1atXLypVqkRYWBh///vfPRydcieLxcLixYsZNGiQVzYrKo5vRat81r///W+eeuopx3KXLl1YuXIlubm5BAUFeTAy5W4Wi4XZs2fTuXNnn0uYoElTuYgxhtjYWBYuXMiCBQvYtWtXge1PPvkkgCbMAJOdnY3VaqV3795FTkzn7TRpKpf44osvuP/++x3Lffr0Ye/evTRu3Jhx48bxu9/9zoPRKU/Iyspi1qxZ3HbbbTRq1MjT4ZSZJk1Vro4ePcrjjz/OwoULC6z/61//SpcuXXTa2gC2bt06brnlFp9OmKBJU5WjuLg4XnjhBUfCDA0NpV69etx666307t1b210GqIyMDPbu3UufPn28auCNstKkqcrF+++/X6AZ0WOPPcZ7773nwYiUN0hPT2fWrFn069fPLxImaNJUV+nMmTPMnj2badOmAdCwYUMSEhI8HJXyBlarlUuXLjFgwADq1avn6XDKjQ4To67Kxx9/zIQJExxvxydOnOjhiJQ3uHDhAtOnT6datWp+lTBBa5qqBLm5uWzcuJGMjAzHuhkzZjB79mxq1KjB0aNHHevfeecdJkyY4IkwlRfJzc1l7ty5DBkyxC+fY2vSVEV67bXXePXVV8nMzCx0e1paGkOHDuWaa67hrbfe8rsahSq91NRUsrOzGTduHCEhIZ4OxyU0aaoCzp49y8WLF3n00Ucdb8E7derEm2++SXh4OEuWLOHVV18FbM2LIiMjPRit8iZnz55lwYIFjBw50m8TJmjSVPls2LCBHj16FFi3efNmbrjhBsfyP/7xDwBiY2M1YaoCDh8+THR0NNWqVfN0KC6lSVMBtudQf/zjHwF44IEHuPnmm+nfv/8Vw3V9+umn7Nixg6ioEmc6VQEiKSmJ7du3M2DAAE+H4haaNBUnT56kYcOGjuX333+fsLCwK8pt2rSJU6dOMXjwYHeGp7xYcnIyP/zwA6NGjfJ0KG6jSVPxzDPPABAcHExKSkqhCRNg5MiRJCYmcv78eSpVquTOEJUXOnfuHFWqVCEmJqbI/zP+SNtpBrjc3FxmzJgB2AZUKG4g4MaNGxMaGqoTmylOnDjBokWLEJGASpigSTOgJSYmOmqM1113XbFT4r799tvExsbSq1cvN0WnvJXVamXz5s3ExMQE5B2H3p4HKIvFUmCGxyVLlhRb/vTp04BtSl0VuOLi4rBYLERHR3s6FI/RmmaAyptjvGnTpmRlZdGgQYNCy1mtVj788EMee+wxTp486dhPBZ4jR47w888/06RJE0+H4lEuTZoiMkhEDojIIRF5rpDt1URkoYjsFJG9InJ/YcdR5S89PR2APXv2FNvV7eWXX2bChAkcOnRI5x4PYDk5OYgIMTExftk1sjRcdnsuIkHAB0B/IAGIFZEFxph9+Yr9CdhnjBkmIrWBAyIy3RiT7aq4lM2CBQto1aoVVapUKXR7VlYW7dq148iRIwA0a9bMneEpL3LgwAEOHDjA8OHDPR2KV3BlTbMbcMgYc8SeBL8BRlxWxgARYhtorwpwFrC4MKaA9+uvv1K1alVycnKK7eq2Z88ejhw5Qr9+/VixYgXNmzd3Y5TKWyQmJrJr1y5uv/12T4fiNVz5IqghEJ9vOQG48bIy/wEWACeBCGCsMcZ6+YFEZDwwHgj45ylXwxjDX/7yFy5cuMD111/PlClTiixbr149XnnlFYYMGUKXLl3cGKXyFgcPHqRJkyaMGjWq2JYVgcaVSbOwYZrNZcsDgR1AH+BaYJmIrDXGnC+wkzFTgakAUVFRlx9DleD8+fMMHjyYuLg4Tp48CcDGjRsLbV9ntVp5+eWX6dmzJxMnTtRflgC1Y8cO4uLiaNGihf4fuIwrk2YCkL/jciNsNcr87gfeNMYY4JCIHAXaAJtdGFfAMMZw+PBhWrZsCUCFChXo1q0bkydPLjRhJiYm0rhxYywW2xOS7du306lTJ3eGrLyAMYb09HSGDx+uCbMQrkyasUBLEWkGnADuBO66rMxxoC+wVkTqAq2BIy6MKSAYY0hISOD6668nLS3NsT4rK4vg4MJ/5Dk5OQXejv/666+0aNHC5bEq7xIbG0toaOgVo12p37jsz4gxxgI8CiwB9gPfGWP2isjDIvKwvdgrQA8R2Q2sAJ41xiS7KqZAMXDgQJo0aeJImN999x0ZGRlFJkywTU/w7LPPMnbsWLKzszVhBqCtW7eSnJzMdddd5+lQvJrY7ox9R1RUlNmyZYunw/BKubm5NGrUiMTERAA++eQT7r33XoKCgord75NPPuHgwYOOsTJV4Dlz5gxhYWGEh4f7zayRpSUiW40xJY55qA8s/MjixYsdCXP79u088MADJSZMgK+++oq33nrL1eEpL7V69Wr27NlDlSpVAjZhloYmTT/yzTffALbnUs68wMnKyqJv376sXr3axZEpb5WUlISI0KdPH0+H4jM0afqRY8eOUalSJadHVd+5cyc//fQTnTp1YunSpS6OTnkTYwyrVq0iLCyMnj17ejocn6KjHPmJxx9/nLVr1xbZLfJyxhjq1q3LF198QceOHenYsaOLI1TewhjDkiVLqFatGuHh4Z4Ox+do0vQDBw4c4L333gNg3rx5xZY1xnDu3DmGDx9Oz549eeWVV5x67qn8gzGG7Oxs2rVrp73rykhvz33cBx98QJs2bQD497//Td++fYsse+bMGWrUqEHNmjVZv349b7zxhj74DyBWq5WFCxdy+PBhTZhXQZOmD9uxYwePPvooAFOnTmX8+PHFlj9y5AipqakAvPvuuxw+fFh7fASQjRs30qRJE9q1a+fpUHya3p77sKNHjwLw/PPP8+CDDxZZzhjDxo0b6dChAx999FGJyVX5l9zcXHbu3En37t31zqIcaDXDR+Xm5rJ8+XIAxo4dW2iZ1157jebNmxMREUGPHj14+umnNWEGmNzcXObOnUtoaKgmzHKiNU0fZLFYaNmyJXFxcQBUq1btijLnz5/nxRdfBH5Lqn/729/cFqPyPGMMFy9epEOHDrRq1crT4fgNTZo+5OOPP2bRokUsWLDAsW7nzp1ERkYWKGexWKhYsSLPPPMMQ4YMoXfv3m6OVHlaTk4Oc+bMYciQIZowy5kmTR8yfvx4IiIi6NixI7m5uaxbt67QWuZNN91EgwYNmDt3rjYnCkDGGObOnUtUVJTOUe8CmjR9xF//+lcAOnTowLp16wotkzf4SmpqKlu3btWEGYCysrJITU1l+PDhhIaGejocv6QvgrycMYaePXvy9ttvAzB9+vQiy44ePZoKFSpw+PBh7r77bneFqLxEZmYmM2fOJCsrSxOmC2nS9HL79+9n7dq1AGzevJmmTZsWWXbgwIGEhYUxadIknn32WXeFqLzE/v376dWrlzZcdzG9PfdiJ0+epH379gB8/fXX3HDDDUWW/ctf/sLrr7/Ogw8+qE1LAkx6ejpr1qxh0KBBng4lIGhN04v9+c9/dnwuqi0mwP/93//xz3/+kw8++EATZoC5dOkSs2bNonPnzp4OJWBo0vRSGzduZPbs2VSpUoWcnJwip6qYOnUqr7zyCkCxNVHlf9LT0wkODub222+nbt26ng4nYGjS9ELGGLp37w7AxIkTi53b56GHHgLgyy+/5Oabb3ZLfMrzzp8/73jpU7NmTU+HE1D0maaXMcY4evpUrlyZ559/vshyVquVDz/8kCpVqvC73/3OjVEqT1uxYgXDhg2jatWqng4l4GjS9CIWi4UaNWpw4cIFwDbUW1EGDRpEWloaGzdudFd4ygucPXuW06dPM3LkSH1+7SF6e+4lrFYrgwcPdiTMyZMn88ADDxRaNj09naVLl3LgwAF3hqg8LDk5mQULFtCwYUNNmB6kNU0vcO7cOdq1a+eYSfLMmTPUrl27yPIjR44EIDo62h3hKS9gtVq5cOECo0aN0q6RHqZJ0wvUqFHD8fnYsWPFJkyAXbt2AfD++++7NC7lHRITE1m9ejVjxozRGqYX0KTpQZmZmQWeSWZkZDjV/e2NN94gPDxcJ8UKABcuXGDp0qWMHj1aE6aX0KTpQZGRkZw+fRqAGTNmOJUwk5OTGTFiRIHaqfJPJ0+epFq1aowbN46QkBBPh6Ps9EWQh1y4cMGRMBcuXMioUaNK3Gfu3LnUrl1bmxcFgOPHj/PTTz8RFBSkCdPLaE3TQ+bPnw/ACy+8wNChQ53aJy9Z/uEPf3BZXMo7xMXFERMTQ6VKlTwdirqMJk0P+fLLLwG48847nSq/bt060tPTAZyqlSrfdPjwYZKTk+nZs6enQ1FFcPr2XET0rUM5OH/+PPXr12fZsmUAjlGMSnLy5EmAIgcgVr7v8OHDxMbG0qVLF0+HoopRYtIUkR4isg/Yb1/uKCL/dXlkfsYYw8SJE6lWrRqJiYkEBwezbNkyp9+IjhkzBmOM9i/3U6mpqdSqVYvRo0frM0wv50xN89/AQCAFwBizE9B7h1JaunQpb7zxBmBLgNnZ2fTr18+pfefPn89NN93EqVOnXBmi8pC9e/eyfPlyqlatWuzgLMo7OHV7boyJv2xVrgti8VsPPvigY4DYr776im+//bZUbe6eeuopNm3a5KrwlAddunSJQ4cOER0dre0wfYQzSTNeRHoARkQqisgz2G/VVcl++uknpk2bBth68IwbN65U+1utVo4cOUKlSpWoX7++K0JUHrJz504yMjIYMWKEToLnQ5y5F3gYeBdoCCQAS4EJrgzKXxhj6Nu3L2C7Pe/fv3+pj5GXZMuyr/JeW7du5eTJk1x//fWeDkWVkjM1zdbGmLuNMXWNMXWMMb8D2ro6MH+QNwpRZGRkqZNe165dyc7O5sknn+Saa67hf//7nytCVB5gsViIiIhg6NChVKig/Ut8jTM/scJGhdCRIpywatUqAMd0FM7av38/27Zt4+GHH+bGG28kJSVFR7bxExs2bGDTpk20atVKn2H6qCJvz0WkO9ADqC0iT+XbVBXQBzBOePHFFwHo3bt3qfa76667ANstuf5i+Y+9e/dy8eJFp1tNKO9U3DPNikAVe5n81ZzzwGhXBuUPLBYLKSkpADRs2NDp/TZv3syOHTsA53sLKe9mjGH//v20bt2adu3a6R9CH1dk0jTGrAZWi8jnxphjbozJL+T14ImJiSnVftWqVQNgzpw5+svlB4wx/PTTT4SEhNCuXTtPh6PKgTNvz9NF5G2gPeAYu8wY08dlUfmBr7/+GoDhw4eXar/WrVvzyy+/0Lp1a1eEpdwsMzOTqlWr6vTKfsSZF0HTgV+AZsDLQBwQ68KY/MLcuXMBuOOOO5zeJyMjg5iYGJKTk10VlnITYwyLFy8mLS1NE6afcSZp1jTGfALkGGNWG2MeAG5ycVw+6+zZs0yePJktW7YAEBYW5vS+q1evZtasWfz000+uCk+5yZIlS6hZsyb16tXzdCiqnDmTNHPs30+JyO0i0hlo5MzBRWSQiBwQkUMi8lwRZXqJyA4R2Ssiq52M22t99dVXPPnkkwA88cQTTu9njOHtt98GcDSIV77HarWSnJzMzTffrDVMP+XMM81XRaQa8DS29plVgSdK2klEgoAPgP7YehLFisgCY8y+fGWqA/8FBhljjotInVJfgZc5fPgwACdOnKBBgwZO77dz505HDbNTp06uCE25mNVqZcGCBTRv3pwOHTp4OhzlIiXWNI0xi4wxacaYPcaY3saYrsBZJ47dDThkjDlijMkGvgFGXFbmLmCOMea4/VxnShm/VzHGOGaIrFmzZqn2PXbsGE2aNOHbb78t1S298h4HDx6kRYsWmjD9XHGN24OAMdj6nC82xuwRkaHARKAy0LmEYzcE8o+OlADceFmZVkCIiKzC1hb0XWPMl6W6Ai9y5owt53fu3LnU0xT06tWLrVu3ljrZKs+zWCysXLmSvn37arfIAFDcT/gT4I9ATeA9EfkM+CfwljGmpIQJUFgjQ3PZcjDQFbgd25idL4lIqysOJDJeRLaIyJakpCQnTu1+sbGxjof+v//970u174kTJ+jUqRMVKlTQtpk+xmKxMHv2bJo0aaIJM0AU90wzCuhgjLGKSCiQDLQwxiQ6eewEoHG+5UbAyULKJBtjLgGXRGQN0BE4mL+QMWYqMBUgKirq8sTrFfLaY1avXp1HHnmkVPvOnz+fuLg4Vq5cqfP/+JCcnBzHaPqNGjn1blT5geL+NGYbY6wAxphM4GApEibY2nK2FJFmIlIRuBNYcFmZ+cCtIhIsImHYbt99bqzOGTNmkJiYSMOGDTl37pxT85fnmT9/Pn/6058AaNasmatCVOUsOzubmTNncvbsWU2YAaa4mmYbEdll/yzAtfZlAYwxptin3cYYi4g8CizBNsDHp8aYvSLysH37FGPMfhFZDOwCrMA0Y8yeq7wmt8sbZPjTTz8t9b4rV64EbI3hdUIt37Fs2TJ69Oih7TADkBhT+N2uiDQtbkdP9UePiooyeQ3HvUVwcDC5ubkU9W9ZnOTkZAYPHkxsrHay8gWZmZkcPnxYB97wQyKy1RgTVVK54gbs0EE6nHDs2DFyc3Np27Zs4zKLCKtX+3yb/oCQkZHBrFmz6NOnjybMAKav+65SYqLtMe/TTz9d6n3nzJlDVFQUixYtKu+wVDkzxpCUlES/fv1KNdSf8j86X+hVyMjI4KabbN3wa9WqVer9896UN27cuISSypMuXrzIwoULGTNmjE6AppyraYpIZRHRscryOX/+vKPnTuPGjR1T9Drr4sWLAFx33XV079693ONT5SM7O5vZs2fTr18/TZgKcCJpisgwYAew2L7cSUQubzoUcFJTUwGIiopi3759pe4B9P333wMwdOjQ8g5NlZO0tDSysrIYM2YMtWvX9nQ4yks4U9OchK0feSqAMWYHEOmqgHzF1q1bAXjooYeoUqVKqfevXr061113HWPHji3v0FQ5OHfuHHPnzsVisVC5cmVPh6O8iDPPNC3GmDR9W1jQq6++ClCmwRk+/vhj2rZty/bt2wkO1sfK3mj37t2MHDmS6tWrezoU5WWc+Y3dIyJ3AUEi0hL4M/Cza8PyfvXr1wegW7dupd53xowZGGMcDduV90hOTmb//v307NnT06EoL+XM7flj2OYHygL+B6ThxHiagaBr166l3uef//wnK1eu5MCBAy6ISF2NM2fO8P333+t4pqpYztQ0WxtjXgBecHUwvmT16tW0anXFgEzFunjxIn/5y18AePzxx10RliqjS5cuERoayujRowkPD/d0OMqLOVPTfEdEfhGRV0Skvcsj8gGXLl3i4sWLxMfHl1z4MjVq1ODzzz/n2WefdUFkqixOnjzJ7NmzCQsL04SpSuTMyO29gV5AEjBVRHaLyIuuDsybTZo0CYDx48c7Vf6///0vzZs3Z8WKFaSkpHDvvfe6MDpVGhaLhTVr1hATE6Mv5ZRTihywo9DCItcDfwXGGmMquiyqYnjDgB15LQlOnz5NnTolT2s0ZMgQfvzxRzZv3qyTbXmRY8eOERwcTIMGDbQvuXJ6wA5nGre3FZFJIrIH+A+2N+cBO4CgxWIBoEWLFk4lTIAff/wRQBOmFzl69Cjr16+ndu3amjBVqThzP/IZMAMYYIy5fOT1gHP+/HkAbr75ZqfKv/TSS4AtySrvYLVaycjIICYmhpCQEE+Ho3xMiUnTGHOTOwLxNc4MGJycnMz48eNJSkpyvDVXnnXw4EGOHDlS6rEClMpT3GyU3xljxojIbgpOiObUyO3+av9+22wcGRkZxZZLTU0lOjqa+++/nylTprgjNFWC+Ph4du7cSXR0tKdDUT6suJpmXkNCHVEin7xkWVID6E2bNrFu3boy9RhS5S8uLo4GDRpwxx136GhF6qoU+SLIGHPK/nGCMeZY/i9ggnvC814ltec7evQoACNGjHBHOKoYu3btcvTz14SprpYzjdv7F7JucHkH4m/ypvHVF0CelZubS3JyMiNGjNB5yVW5KPJ/kYg8Yn+e2VpEduX7Oopt9khVhEuXLtG6tW3M5gYNGng4msC1ZcsWjh49Sp8+fTRhqnJT3DPN/wE/Am8Az+Vbf8EYc9alUfm48PBwtm/frreCHhQbG0tSUlKZBlVRqjjFJU1jjIkTkT9dvkFEagRq4nRmdKIffviBs2fP8rvf/c4NEanLpaWl0bx5c6KiorThuip3xc17vsgYM9R+O26wNTXKY4wxzd0R4OU83Y2yevXqpKWlcezYMZo0aVJombCwMJo1a8bevXvdHJ1au3YtFouF3r17ezoU5WPKY97zofbvzcozMF82ceJE0tLSAGjUqPCepDNmzCAjI4OUlBR3hqaAhIQELBYLvXr18nQoyo850/f8ZhEJt3/+nYi8IyKFV7H8XFxcHGAbrLawFws5OTncddddADoquxsZY/j555+pXbs2vXv31lty5VLOvFL8EEgXkY7YRjg6Bnzl0qi81IwZM2jUqFGRMxOeOnWK22+/nWrVqtG2bVs3RxeYjDEsW7YMoNQzgipVFs5OrGZEZATwrjHmExEJuAEh8+YpT0xMLLJMkyZNmDdvnr41dxNjDBaLhcjIyFKPoq9UWTlT07wgIs8Dvwe+F5EgIOCGhvnPf/4DwGuvvVbo9kuXLnHNNdfw3//+V28P3cAYw6JFi4iLi9OEqdzKmaQ5Ftukag8YYxKBhsDbLo3KCz3//PNA0aO1v/jii6SmpvLzzwE/UadbrF69mkaNGtGyZUtPh6ICjFMjt4tIXSBvBN3NxpgzLo2qGJ5ochQfH0+TJk1o1KhRofMCWSwWGjRoQFJSEpmZmfpszYWsVisHDx6kZcuW+hhElavyHLl9DLAZiAHGAJtEZPTVh+g7nnvO1iHq73//e6Hb77nnHpKSkggNDdWE6UK5ubnMnTsXi8WiCVN5TIk1TRHZCfTPq12KSG1guTGmoxviu4K7a5pWq9XxC3rp0iXCwsKuKGOMYebMmXTu3FlvF13o9OnTnD17VlsmKJe46sbt+VS47HY8BeeehfqF+fPnA9CjR49CE+YzzzxD8+bNmTAh4EfLcxmLxcL8+fO5/fbbqVu3rqfDUQHOmeS3WESWiMh9InIf8D3wg2vD8h7vvfdege/57du3j3/961988MEH7g4rYFitVmbNmkXHjh0JDQ31dDhKOTVH0F9E5A7gFmz9z6caY+a6PDIvsWrVKqDwkdr//e9/A3D33Xe7MaLAkZ2dTUZGBkOGDKFq1aqeDkcpoPjxNFuKyHz71L0xwL+MMU8GUsLM061bt0JfPOR1lXz66afdHZLfy8rK4rvvvuP8+fOaMJVXKe72/FNgETAK2Aq875aIvMjp06cB6NCh8DnkqlatSrdu3fSNuQts27aN2267jcaNG3s6FKUKKO72PMIY87H98wER2eaOgLzJmjVrALjuuuuu2Jabm8uGDRs4c8ZjTVb9Unp6OrGxsdx2222eDkWpQhVX0wwVkc4i0kVEugCVL1v2ey+88ALAFWMzpqenU79+faZNm6Y1oXJ06dIlZs6c6ZgqRClvVFxN8xTwTr7lxHzLBujjqqC8RUJCAgDXX399gfXJyckkJSU55kBXVy87OxtjDIMHD6ZOnTqeDkepIhU3CHFAD329efNmMjIymDBhwhUDcLzxxhsA2si6nJw/f545c+YwZswYTZjK6wVMI/XSOHfuHK+//joA/fr1u2L7lClTALj//vvdGpc/Msbw/fffM2zYsEI7DyjlbZzpEVRmIjIIeBcIAqYZY94sotwNwEZgrDFmlitjckbNmjUxxhAUFER0dPQV20eOHEliYqL+kl+lc+fOkZqaytixY3WKXeUzXPY/1T7u5gfAYKAdME5E2hVR7h/AElfFUhqxsbHk9cc/dOjQFdsvXLjAN998w4YNG9wdml9JSUlh/vz5XHPNNZowlU9xZpQjsc8N9H/25SYi0s2JY3cDDhljjhhjsoFvgBGFlHsMmA14RdudhQsXAvD9998TGRlZYFtqaipVq1bl7bffxpkh9VThjDGcPn2aO+64g+rVq3s6HKVKxZk/8f8FugPj7MsXsNUgS9IQyD/4ZIJ9nYOINASigSlOHM8t8tpmFjYF7DXXXAPYXhLp6Oxlc/r0aebPn0+7du20p4/ySc4kzRuNMX8CMgGMMeeAik7sV1hWubx6Nhl41hiTW+yBRMaLyBYR2ZKUlOTEqcvGYrGwevVqACpXrlxg2/Hjxx2fZ83y+GNXn3T27FmWLFlC//79PR2KUmXmzIugHPtzRwOO8TStTuyXAORv+d0IOHlZmSjgG3utrRYwREQsxph5+QsZY6YCU8E2nqYT5y6TvL7kl9+Ww28Tq02dOpWKFZ35m6HyO3PmDNWrV2fMmDE6WpHyac7UNN8D5gJ1ROQ1YB3wuhP7xQItRaSZiFQE7gQW5C9gjGlmjIk0xkQCs4AJlydMd9q8eTMA33zzzRXb2rVrxxNPPMGYMWPcHZbPi4+PZ8mSJVitVk2Yyuc5MzTcdBHZCvTFdss90hhTYlcYY4xFRB7F9lY8CPjUGLNXRB62b/ea55h58poQtWnTpsD6119/nU6dOvHOO+/os8xSMsawd+9exowZowObKL9QYtIUkSZAOrAw/zpjzPGi97IxxvzAZQMWF5UsjTH3lXQ8T8jOznb0QV+7di233HKLhyPyHUeOHOHixYsMGjTI06EoVW6ceab5PbbnmQKEAs2AA0B7F8blEfPmzbti3a+//grYJk/ThOm8w4cPs3nzZkaPDqg5+FQAcOb2vMBoFfYRjh5yWUQecvToUUdzo/DwcMf6bdtsI+INHTrUI3H5ovT0dCIiIoiJiSE42KWdzpRyu1J3xTDGbOO3OdD9Rl4/8vfee6/AL/rOnTsJCwujefPmngrNp+zfv5/FixdTp04dTZjKLzkzhe9T+RYrAF2AmsaYga4MrCiumMJ3w4YN9OjRAyg4Te+mTZvo1q2bvvxx0rlz51i5ciUjRozQecmVz3F2Cl9napoR+b4qYXvGWVh3SJ+VlzDvvfdeR8Jcv349N910U6EDdqgr7d27l+DgYKKjozVhKr9W7P2TvVF7FWPMX9wUj9sdOXIEgMGDB/PZZ58BtmYyeS99xo4d67HYfMX27duJj4+nbdu2WitXfq+42SiD7d0b/Xpqi/fft80Xd8899zh+4fOaGHXo0EGTZgmsVisiwrBhw3S0IhUQiqtpbsaWMHeIyAJgJnApb6MxZo6LY3O5pKQkJk+eDEDXrl0d66OibI815s2bp4mgGJs2baJChQrccIPfvRdUqkjOvN6sAaRgmxMor72mAXw+aeb9sj/xxBO0bNnSsX7AgAHs2bOHZs2aeSo0r7djxw5SU1MZMGCAp0NRyq2KfHsuIgnYJlLLS5L5H1YZY8w7he7oYuX19jw9Pd3RHjPvFhPg4MGDfPDBB7z11lva7a8IR44coVGjRoSEhOgzTOU3yuPteRBQxf4Vke9z3pdPy5sD6Omnny7wi797927ee+89yrtZkz8wxvDTTz9x/PhxTZgqYBU7ha8x5u9ui8RD3nyz4LRFjz76KIAOkFuICxcuEBoa6miipVQgKq6m6ffViKCgoCt6rWRkZABXznUeyIwxLF26lNzcXE2YKuAVlzT7ui0KN9uxYwevvfYaubkFB4zPysoiLS2NwYMHeygy77R48WKqV6/umO5DqUBW5O25MeasOwNxpwULbGMh33nnnQXWiwiTJ0+mc+fOngjL6xhjuHjxIjfccAO1atXydDhKeYWAHlFh+vTpBZZ37drFY489pm0zsbUoWLhwIc2bN9dHFUrlE3DZ4ZdffuFvf/vbFestFgs9evSgWzdnZif2fzt37qRZs2aaMJW6TMDVNOfPnw/AI488UqBGuX37dnJycmjQoIGnQvMKubm5/Pzzz9xyyy3apEipQgRcTfPdd98FbL2A8rt0ydZDdMKECe4OyWtYLBZmz55N7dq1NWEqVYSAq2mGhoYycOBAWrVqVWD94sWLAQJ2et7c3FxycnK44YYbtPuoUsUIuJpmhQoVCn0TnJOTQ+PGjR2DdQSS7Oxsvv32W9LT0zVhKlWCgEuaRfnHP/7B0qVLA7In0KJFi+jevTs1a9b0dChKeT1NmkB8fDyPPPJIwL0EyszM5OjRowwfPlxrmEo5KeCS5uHDh69Yd/DgQaZNm8bq1as9EJFnZGRkMHPmTEJCQnQCNKVKIaB+Ww4cOADA6dOnC6yfNm0aAFWq+PzgTU47fvw4ffv2DbjatVJXK6BqmvmntsgvNjYWgF69erk7JLe7dOkSc+bMoVWrVpowlSqDgEqaqampAIwYUXAyzeTkZMLCwvy+bWJGRgazZs3ShutKXYWAuT03xjB9+nSaNWt2xRvyl156iXr16nkoMve4cOECFStWJDo6OiBbCChVXgKmprlnzx7gt/Ey88TGxhITE8Pdd9/tibDcIjU1ldmzZ5ORkaEJU6mrFDBJ88KFCwB88MEHjnUHDhygW7dujmed/mrTpk2MHDmS6tWrezoUpXxewNye58n/hrxNmzaAbX5zf5SSksLRo0cZOHCgp0NRym8ERE3TarXSs2dPAMcLkLw35m3btuX3v/+9x2JzlaSkJBYtWkTr1q09HYpSfiUgapozZ850TG3RvXt3wNagHX4b9cifZGdnY7VaGTVqVEC1PVXKHQIiaR46dAiwDaybl0Sio6M5e/as3817c+rUKVasWMFdd92lI9Ar5QIB8Vv14osvAtC4cWOys7OpX78+kyZN8ruEmZmZyYoVKxg1apQmTKVcxO9rmsePHwdsL4CCg4OpVKkS8Nugw/4iPj6esLAwrWEq5WJ+/9uV1wvo/fffd0x1AfDyyy97KKLyFxcXx5o1a6hSpYomTKVczO9/w3bu3AlAeHg4p06dAmwN3f1lSlpjDMnJycTExDhq0Uop1/H72/O8t+bt2rVj4MCBjB07lkaNGnk4qvLx66+/cuLEiYAYaEQpb+H3Nc08ubm5xMbGEh4e7he3sEePHmXbtm3ccsstng5FqYDi+9nDSfv27aNfv35s3brV06FctVOnTlG/fn1Gjx6tAwgr5WZ+nzQtFgsA9913H4DPJ5ndu3fz888/U7FiRYKCgjwdjlIBx++T5qxZswDIysqievXqju6Uvig7O5sTJ04QHR3tF48YlPJFvl3tKsGUKVNYsmSJY3nFihU+W9Pctm0bderUYdCgQZ4ORamA5tLqiogMEpEDInJIRJ4rZPvdIrLL/vWziHQsz/PnzQk0a9Ys1q5dS5cuXcrz8G4TGxvLyZMnadiwoadDUSrguazaJSJBwAdAfyABiBWRBcaYffmKHQVuM8acE5HBwFTgxvKKISgoiPDwcEaNGlVeh3S7jIwMGjRoQFRUlE5RoZQXcGVNsxtwyBhzxBiTDXwDFJicxxjzszHmnH1xI1CuDSjj4+PJysrilltu4aWXXirPQ7vFunXriI2NpWHDhpowlfISrkyaDYH4fMsJ9nVF+QPwY2EbRGS8iGwRkS1JSUlOB7B+/XosFgvr169n+fLlTu/nDQ4dOkR2dja33nqrp0NRSuXjyqRZWNXIFFpQpDe2pPlsYduNMVONMVHGmKjatWs7HcCJEyccIxm9+eabTu/nScYYtmzZQmRkJL1799YaplJexpVJMwFonG+5EXDy8kIi0gGYBowwxqSUZwBhYWHUqVMHgKZNm5bnoV3CGMPy5cvJzMwkODhYE6ZSXsiV7W9igZYi0gw4AdwJ3JW/gIg0AeYAvzfGHCzvAEJCQjh//jwAdevWLe/DlytjDBaLhbp16/rtnEVK+QOX1TSNMRbgUWAJsB/4zhizV0QeFpGH7cX+D6gJ/FdEdojIlvKOIyYmhv3791O5cuXyPnS5Mcbwww8/cOLECU2YSnk5l7b0Nsb8APxw2bop+T7/EfijC8/P+fPnvX5Uo+XLl1OvXj0iIyM9HYpSqgR+3RfParXy+eefs2jRIk+HUiir1crx48fp2bMnXbt29XQ4Sikn+H3SBFuN09vk5uYyb9480tLSdPBgpXyIb3bEdlJesqxXr56HI7lSQkICbdq0oV27dp4ORSlVCn5d08zjTX22LRYLCxcupEGDBpowlfJBfp0088bS9JbbX4vFwqxZs2jTpg0hISGeDkcpVQZ+fXteuXJlYmJivKJhe3Z2Nrm5ufTt25fS9GpSSnkXv61pPvvss2RmZnpFgsrKymLmzJmkpKR4RTxKqbLz25rmW2+9Bfz2Bt2T1q9fz6233ur17UWVUiXzy6SZlpbm+BwWFuaxODIyMti9ezd9+vTxWAxKqfLll7fn+Qcd9tQb6vT0dGbOnEnjxo1LLqyU8hl+mTRbtWrl0fPn5uaSnp7OwIEDqV+/vkdjUUqVL79Mmm3btnV8dveLlwsXLjB9+nSqVq3q9SMrKaVKzy+T5hNPPAHAggUL3Dp7Y17XyCFDhlCxYkW3nVcp5T5++SJIRIiIiGDYsGFuO2dqaiqZmZmMGzfOZ6cJVkqVzO9+u/Makbuzv/nZs2dZsGABI0eO1ISpfEJOTg4JCQlkZmZ6OhS3Cw0NpVGjRmXuled3v+GpqamAbfpedzl69CjR0dFUq1bNbedU6mokJCQQERFBZGRkQE2rYowhJSWFhIQEmjVrVqZj+NUzTavVyqOPPgpAjRo1XH6+M2fOsGTJErp27aoJU/mUzMxMatasGVAJE2yP7mrWrHlVNWy/Spo5OTm0bt0awOXTRiQnJ/Pjjz9y8803u/Q8SrlKoCXMPFd73X6VNCtVqsRjjz0GwPXXX++y85w9e5aIiAhiYmKoUqWKy86jlD/z1d8dv0qaJ0+e5H//+59Lz5GQkMD3338PeLaLplKBKm/IR0/xq6T51Vdf8eSTT7rs+Farla1btxITE+M1Y3Qq5U8WLlzIjTfeSOfOnenXrx+nT58GYNKkSYwfP54BAwZwzz33kJSURP/+/enSpQsPPfQQTZs2JTk5GYCvv/6abt260alTJx566CFyc3PLNUa/enuemJjosmPHxcWRk5PDiBEjXHYOpTzhiSeeYMeOHeV6zE6dOjF58uRS73fLLbewceNGRIRp06bx1ltv8a9//QuArVu3sm7dOipXrsyjjz5Knz59eP7551m8eDFTp04FYP/+/Xz77besX7+ekJAQJkyYwPTp07nnnnvK7dr8KmkePXrU8blmzZrldtzDhw+zefPmAgOBKKXKX0JCAmPHjuXUqVNkZ2cXaBY0fPhwKleuDMC6deuYO3cuAIMGDeKaa64BYMWKFWzdupUbbrgBsI00VqdOnXKN0a+SZnh4OJUqVSIrK4vo6OhyOWZOTg7BwcGMHj1ap6hQfqksNUJXeeyxx3jqqacYPnw4q1atYtKkSY5t4eHhjs9FzTBrjOHee+/ljTfecFmMfvVMs0WLFlStWhWgXBLcL7/8wo8//kjTpk01YSrlBmlpaY6JEL/44osiy91yyy189913ACxdupRz584B0LdvX2bNmsWZM2cAW0uXY8eOlWuMfpU0n3/+eZKSkmjatOlVt8U6ffo0e/bs4fbbby+n6JRS+aWnp9OoUSPH1zvvvMOkSZOIiYnh1ltvpVatWkXu+7e//Y2lS5fSpUsXfvzxR+rXr09ERATt2rXj1VdfZcCAAXTo0IH+/ftz6tSpco3bb27PjTHEx8cDMHDgwKs61oEDB2jatCl33HEHFSr41d8VpbxGUVPRFPayNf9tOkC1atVYsmQJwcHBbNiwgZUrVzpatIwdO5axY8eWe7x5/CZpZmZmOgYfbtOmTZmPs2PHDuLi4mjZsqUmTKW81PHjxxkzZgxWq5WKFSvy8ccfu+3cfpM0V69e7fjctWvXMh3DGENGRgbDhw/XhKmUF2vZsiXbt2/3yLn9Jmn++uuvV7X/5s2bCQ0NpXv37uUUkVLKH2l1Cluj2bNnz7q0v7pSyj/4TU0zv9Ikv8TERNq0aUOXLl0CdtQXpZTz/Kammfe2bPTo0Y7eAcUxxrB69Wr2799PeHi4JkyllFP8JmnmPdOMiIhwqnxKSgoVKlSgd+/ergxLKVWM1157jfbt29OhQwc6derEpk2brvqYeUPOxcXFcd1111318S7nN7fnK1euBGDcuHHFljPGsGrVKrp168att97qjtCUUoXYsGEDixYtYtu2bVSqVInk5GSys7M9HVaJ/KamuWLFihLLGGNYvHgxlStXLtCPVSnlfqdOnaJWrVqORum1atWiQYMGREZGMnHiRLp3705UVBTbtm1j4MCBXHvttUyZMgWAixcv0rdvX7p06cL111/P/Pnz3Ra339Q09+3bB1BkMjTGkJWVxXXXXUfjxo3dGZpSXq9Xr15XrBszZgwTJkwgPT2dIUOGXLH9vvvu47777iM5OZnRo0cX2LZq1aoSzzlgwAD+/ve/06pVK/r168fYsWO57bbbAGjcuDEbNmzgySef5L777mP9+vVkZmbSvn17Hn74YUJDQ5k7dy5Vq1YlOTmZm266ieHDh7vl3YRfJE2r1erooN+pU6dCty9atIgWLVrQrl07N0enlCpMlSpV2Lp1K2vXrmXlypWMHTuWN998E7ANAwe2ljAXL14kIiKCiIgIQkNDSU1NJTw8nIkTJ7JmzRoqVKjAiRMnOH36tFum7vaLpLl3717ANrJRYVNQbNq0iaZNm2rCVKoIxdUMw8LCit1eq1Ytp2qWhQkKCqJXr1706tWL66+/3jGyUd4te4UKFQrMklChQgUsFgvTp08nKSmJrVu3EhISQmRkpNvmcPeLpJmUlARc+RIoNzeXHTt2cNNNN2mTIqW8zIEDB6hQoQItW7YEbOM+NG3alN27d5e4b1paGnXq1CEkJISVK1eW+/BvxfGLF0F58wKNHDnSsc5isTBnzhwqV66sCVMpL3Tx4kXuvfde2rVrR4cOHdi3b98VoxkV5e6772bLli1ERUUxffr0qxqkp7SkqBGQvVVUVJTZsmVLgXV5STErK4uKFStijOH8+fOcOXPG8VdMKfWb/fv307ZtW0+H4TGFXb+IbDXGRJW0r1/UNAFCQ0OpWLEiOTk5fPvtt4iIJkylVLlzadIUkUEickBEDonIc4VsFxF5z759l4h0Keu5goODMcYwd+5cunXr5pj2QimlypPLXgSJSBDwAdAfSABiRWSBMWZfvmKDgZb2rxuBD+3fSy03N5fTp08zfPhwQkNDry54pZQqgitrmt2AQ8aYI8aYbOAb4PJx7EcAXxqbjUB1EalfmpMcPnwYsD3PzMnJ0YSplJN87X1Gebna63Zl0mwIxOdbTrCvK22ZYl28eBGwTXGhPX2Uck5oaCgpKSkBlziNMaSkpFxV5cqV7TQLa+dz+U/ImTKIyHhgPECTJk0KbGvatCkffPABffv2LWOYSgWeRo0akZCQ4GjjHEhCQ0Np1KhRmfd3ZdJMAPJX/RoBJ8tQBmPMVGAq2Joc5d9WvXp1JkyYUB7xKhUwQkJCaNasmafD8EmuvD2PBVqKSDMRqQjcCSy4rMwC4B77W/SbgDRjTPlOUqyUUuXIZTVNY4xFRB4FlgBBwKfGmL0i8rB9+xTgB2AIcAhIB+53VTxKKVUeXNr33BjzA7bEmH/dlHyfDfAnV8aglFLlyee6UYpIElBY7/xaQLKbw3E1f7wm0OvyJf54TVD4dTU1xtQuaUefS5pFEZEtzvQb9SX+eE2g1+VL/PGa4Oquy2/6niullDto0lRKqVLwp6Q51dMBuIA/XhPodfkSf7wmuIrr8ptnmkop5Q7+VNNUSimX86mk6c7xOd3Jieu62349u0TkZxHp6Ik4S6Oka8pX7gYRyRWR0UWV8SbOXJeI9BKRHSKyV0RWuzvGsnDi/2A1EVkoIjvt1+X1HVFE5FMROSMie4rYXrZ8YYzxiS9svYoOA82BisBOoN1lZYYAP2IbCOQmYJOn4y6n6+oBXGP/PNjbr8uZa8pX7idsHSBGezrucvpZVQf2AU3sy3U8HXc5XddE4B/2z7WBs0BFT8dewnX1BLoAe4rYXqZ84Us1TbeMz+kBJV6XMeZnY8w5++JGbAObeDNnflYAjwGzgTPuDO4qOHNddwFzjDHHAYwxvnBtzlyXASLENiFXFWxJ0+LeMEvHGLMGW5xFKVO+8KWk6ZbxOT2gtDH/AdtfR29W4jWJSEMgGpiC73DmZ9UKuEZEVonIVhG5x23RlZ0z1/UfoC22Uch2A48bY6zuCc9lypQvfGne83Ibn9PLOB2ziPTGljRvcWlEV8+Za5oMPGuMyfWhKZadua5goCvQF6gMbBCRjcaYg64O7io4c10DgR1AH+BaYJmIrDXGnHdxbK5UpnzhS0mz3Mbn9DJOxSwiHYBpwGBjTIqbYisrZ64pCvjGnjBrAUNExGKMmeeWCMvG2f+DycaYS8AlEVkDdAS8OWk6c133A28a28PAQyJyFGgDbHZPiC5Rtnzh6Ye1pXioGwwcAZrx28Pq9peVuZ2CD3Y3ezrucrquJtiGz+vh6XjL65ouK/85vvEiyJmfVVtghb1sGLAHuM7TsZfDdX0ITLJ/rgucAGp5OnYnri2Sol8ElSlf+ExN0/jp+JxOXtf/ATWB/9prZhbjxYMoOHlNPseZ6zLG7BeRxcAuwApMM8YU2uTFWzj583oF+FxEdmNLMs8aY7x69CMRmQH0AmqJSALwNyAEri5faI8gpZQqBV96e66UUh6nSVMppUpBk6ZSSpWCJk2llCoFTZpKKVUKmjSVU+wjEe3I9xVZTNmL5XC+z0XkqP1c20SkexmOMU1E2tk/T7xs289XG6P9OHn/LnvsowBVL6F8JxEZUh7nVp6hTY6UU0TkojGmSnmXLeYYnwOLjDGzRGQA8E9jTIerON5Vx1TScUXkC+CgMea1YsrfB0QZYx4t71iUe2hNU5WJiFQRkRX2WuBuEbliFCMRqS8ia/LVxG61rx8gIhvs+84UkZKS2RqghX3fp+zH2iMiT9jXhYvI9/axHveIyFj7+lUiEiUibwKV7XFMt2+7aP/+bf6an72GO0pEgkTkbRGJtY+1+JAT/ywbsA/4ICLdxDb26Xb799YiUhH4OzDWHstYe+yf2s+zvbB/R+VlPN3NSb984wvIxTZgww5gLraud1Xt22ph61WRd+dy0f79aeAF++cgIMJedg0Qbl//LPB/hZzvc+xdK4EYYBO2gTB2A+HYhifbC3QGRgEf59u3mv37Kmy1OkdM+crkxRgNfGH/XBHbqDeVgfHAi/b1lYAtQLNC4ryY7/pmAoPsy1WBYPvnfsBs++f7gP/k2/914Hf2z9Wx9VEP9/TPW7+K/vKZbpTK4zKMMZ3yFkQkBHhdRHpi6y7YEFuf5MR8+8QCn9rLzjPG7BCR24B2wHp7l9CK2GpohXlbRF4EkrCN7tQXmGtsg2EgInOAW4HFwD9F5B/YbunXluK6fgTeE5FKwCBgjTEmw/5IoIP8NqJ8NaAlcPSy/SuLyA5sfZy3Asvylf9CRFpiGzknpIjzDwCGi8gz9uVQbGMN7C/FNSg30qSpyupubCN4dzXG5IhIHLZfeAdjzBp7Ur0d+EpE3gbOAcuMMeOcOMdfjDGz8hZEpF9hhYwxB0WkK7Z+xG+IyFJjzN+duQhjTKaIrMI29NlYYEbe6YDHjDFLSjhEhjGmk4hUAxYBfwLew9ZXe6UxJtr+0mxVEfsLMMoYc8CZeJXn6TNNVVbVgDP2hNkbaHp5ARFpai/zMfAJtqkHNgI3i0jeM8owEWnl5DnXACPt+4Rju7VeKyINgHRjzNfAP+3nuVyOvcZbmG+wDdZwK7ZBK7B/fyRvHxFpZT9noYwxacCfgWfs+1TDNhIQ2G7J81zA9pgizxLgMbFXu0Wkc1HnUN5Bk6Yqq+lAlIhswVbr/KWQMr2AHSKyHdtzx3eNMUnYksgMEdmFLYm2ceaExpht2J51bsb2jHOaMWY7cD2w2X6b/ALwaiG7TwV25b0IusxSbPPJLDe26R7ANnbpPmCb2Cbm+ogS7szssewE7gTewlbrXY/teWeelUC7vBdB2GqkIfbY9tiXlRfTJkdKKVUKWtNUSqlS0KSplFKloElTKaVKQZOmUkqVgiZNpZQqBU2aSilVCpo0lVKqFDRpKqVUKfw/ilB1+trz5BIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fpr_large, tpr_large, _ = roc_curve(y_val, y_pred)\n",
    "fpr_small, tpr_small, _ = roc_curve(y_val, y_pred_small)\n",
    "\n",
    "plt.figure(figsize=(5, 5))\n",
    "\n",
    "plt.plot(fpr_large, tpr_large, color='black', linestyle='solid', label='Large')\n",
    "plt.plot(fpr_small, tpr_small, color='black', linestyle='dashed', label='Small')\n",
    "plt.plot([0, 1], [0, 1], color='black', lw=0.7, linestyle='dashed', alpha=0.5)\n",
    "\n",
    "plt.xlim([-0.02, 1.02])\n",
    "plt.ylim([-0.02, 1.02])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "\n",
    "plt.title('ROC curve')\n",
    "plt.legend(loc='lower right')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8363381374257972"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8117942866042492"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_val, y_pred_small)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interpretation of AUC: the probability that a randomly chosen positive example\n",
    "ranks higher than a randomly chosen negative example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8356"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg = y_pred[y_val == 0]\n",
    "pos = y_pred[y_val == 1]\n",
    "\n",
    "np.random.seed(1)\n",
    "neg_choice = np.random.randint(low=0, high=len(neg), size=10000)\n",
    "pos_choice = np.random.randint(low=0, high=len(pos), size=10000)\n",
    "(pos[pos_choice] > neg[neg_choice]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-fold cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(df, y):\n",
    "    cat = df[categorical + numerical].to_dict(orient='records')\n",
    "    \n",
    "    dv = DictVectorizer(sparse=False)\n",
    "    dv.fit(cat)\n",
    "\n",
    "    X = dv.transform(cat)\n",
    "\n",
    "    model = LogisticRegression(solver='liblinear')\n",
    "    model.fit(X, y)\n",
    "\n",
    "    return dv, model\n",
    "\n",
    "\n",
    "def predict(df, dv, model):\n",
    "    cat = df[categorical + numerical].to_dict(orient='records')\n",
    "    \n",
    "    X = dv.transform(cat)\n",
    "\n",
    "    y_pred = model.predict_proba(X)[:, 1]\n",
    "\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "kfold = KFold(n_splits=10, shuffle=True, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "aucs = []\n",
    "\n",
    "for train_idx, val_idx in kfold.split(df_train_full):\n",
    "    df_train = df_train_full.iloc[train_idx]\n",
    "    y_train = df_train.churn.values\n",
    "\n",
    "    df_val = df_train_full.iloc[val_idx]\n",
    "    y_val = df_val.churn.values\n",
    "\n",
    "    dv, model = train(df_train, y_train)\n",
    "    y_pred = predict(df_val, dv, model)\n",
    "\n",
    "    rocauc = roc_auc_score(y_val, y_pred)\n",
    "    aucs.append(rocauc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.849, 0.841, 0.859, 0.833, 0.824, 0.842, 0.844, 0.822, 0.845,\n",
       "       0.861])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(aucs).round(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc = 0.842 ± 0.012\n"
     ]
    }
   ],
   "source": [
    "print('auc = %0.3f ± %0.3f' % (np.mean(aucs), np.std(aucs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tuning the parameter `C`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(df, y, C=1.0):\n",
    "    cat = df[categorical + numerical].to_dict(orient='records')\n",
    "    \n",
    "    dv = DictVectorizer(sparse=False)\n",
    "    dv.fit(cat)\n",
    "\n",
    "    X = dv.transform(cat)\n",
    "\n",
    "    model = LogisticRegression(solver='liblinear', C=C)\n",
    "    model.fit(X, y)\n",
    "\n",
    "    return dv, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C=0.001, auc = 0.825 ± 0.013\n",
      "C=0.01, auc = 0.839 ± 0.009\n",
      "C=0.1, auc = 0.841 ± 0.007\n",
      "C=0.5, auc = 0.841 ± 0.007\n",
      "C=1, auc = 0.841 ± 0.007\n",
      "C=10, auc = 0.841 ± 0.007\n"
     ]
    }
   ],
   "source": [
    "nfolds = 5\n",
    "kfold = KFold(n_splits=nfolds, shuffle=True, random_state=1)\n",
    "\n",
    "for C in [0.001, 0.01, 0.1, 0.5, 1, 10]:\n",
    "    aucs = []\n",
    "\n",
    "    for train_idx, val_idx in kfold.split(df_train_full):\n",
    "        df_train = df_train_full.iloc[train_idx]\n",
    "        df_val = df_train_full.iloc[val_idx]\n",
    "\n",
    "        y_train = df_train.churn.values\n",
    "        y_val = df_val.churn.values\n",
    "\n",
    "        dv, model = train(df_train, y_train, C=C)\n",
    "        y_pred = predict(df_val, dv, model)\n",
    "        \n",
    "        auc = roc_auc_score(y_val, y_pred)\n",
    "        aucs.append(auc)\n",
    "\n",
    "    print('C=%s, auc = %0.3f ± %0.3f' % (C, np.mean(aucs), np.std(aucs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Full retrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc = 0.858\n"
     ]
    }
   ],
   "source": [
    "y_train = df_train_full.churn.values\n",
    "y_test = df_test.churn.values\n",
    "\n",
    "dv, model = train(df_train_full, y_train, C=0.5)\n",
    "y_pred = predict(df_test, dv, model)\n",
    "\n",
    "auc = roc_auc_score(y_test, y_pred)\n",
    "print('auc = %.3f' % auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}