spatiallagmodles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7c6fa5b8-58be-4097-91ab-2d7c34867118",
   "metadata": {},
   "source": [
    "## Table of Contents\n",
    "- [Loading X_train, X_val and X_test](#section1)\n",
    "- [Add spatial lag features](#section2)\n",
    "- [Encoding categorical features](#section3)\n",
    "- [KNN](#section4)\n",
    "- [Distanceband](#section5)\n",
    "- [ROOK](#section6)\n",
    "- [Predictions](#section7)\n",
    "- [Classification_report + metrics ](#section8)\n",
    "- [Province analysis (error count)](#section9)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60f42bdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import scipy.stats\n",
    "\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "%matplotlib inline\n",
    "import geopandas as gpd\n",
    "import torch\n",
    "import joblib\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, balanced_accuracy_score,confusion_matrix, classification_report\n",
    "np.random.seed(10)\n",
    "from libpysal import weights\n",
    "\n",
    "# import self written function\n",
    "from functions_spatialmodels import Objective_knn, Objective_rf, Objective_xgb, Objective_tabnet, all_metrics, predict, conf_matrix\n",
    "from functions_spatialmodels import spatial_lag_distance_band, spatial_lag_knn, spatial_lag_queen, spatial_lag_rook, encoding_categorical_features, scaling_features\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "774d3725",
   "metadata": {},
   "source": [
    "<a id='section1'></a>\n",
    "# Loading X_train, X_val and X_test\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "04ad1595-fdce-496f-af4d-cbbbe523cbc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = pd.read_csv(\"X_test.csv\", index_col=0)\n",
    "X_train_split= pd.read_csv(\"X_train_split.csv\", index_col=0)\n",
    "X_val= pd.read_csv(\"X_val.csv\",  index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9db887c-cf28-4699-86cb-a19d9ea6aec9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'geopandas.geodataframe.GeoDataFrame'>\n",
      "RangeIndex: 4070 entries, 0 to 4069\n",
      "Data columns (total 2 columns):\n",
      " #   Column     Non-Null Count  Dtype   \n",
      "---  ------     --------------  -----   \n",
      " 0   postcode4  4070 non-null   int64   \n",
      " 1   geometry   4070 non-null   geometry\n",
      "dtypes: geometry(1), int64(1)\n",
      "memory usage: 63.7 KB\n"
     ]
    }
   ],
   "source": [
    "# Source: https://www.cbs.nl/nl-nl/dossier/nederland-regionaal/geografische-data/gegevens-per-postcode\n",
    "# extracting geometry code of each postcode\n",
    "gpkg = gpd.read_file(\"cbs_pc4_2022_v1.gpkg\")\n",
    "gpkg[\"postcode4\"]= gpkg[\"postcode4\"].astype(\"int64\")\n",
    "gpkg = gpkg[[\"postcode4\", \"geometry\"]]\n",
    "gpkg.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "439ab138-3f3c-4907-b6d2-b30b905ec0f3",
   "metadata": {},
   "source": [
    "<a id='section2'></a>\n",
    "# Add spatial lag features \n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "90a53d7d-7a88-409e-a471-3bab16380192",
   "metadata": {},
   "outputs": [],
   "source": [
    "lst =[\"SurroundingAddressDensity\",\"AvgDistToTrainStation\",\"NumBusStops\",\n",
    "    'RoadNetworkDensity',\"AvgDistToSupermarket\",\n",
    "   \"NofBusinessEstablishments\"] # extract built environment features for spatial analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2b29d47e-8291-42b7-affb-b28274f10fd8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_KNN8 = spatial_lag_knn(gpkg, X_train_split, 8, lst, \"LagKNN8\")\n",
    "X_test_KNN8 = spatial_lag_knn(gpkg, X_test, 8, lst, \"LagKNN8\")\n",
    "X_val_KNN8 = spatial_lag_knn(gpkg, X_val, 8, lst, \"LagKNN8\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "37478851-2360-4d45-a508-540abb1f3e25",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_rook = spatial_lag_rook(gpkg, X_train_split, lst, \"LagRook\")\n",
    "X_test_rook = spatial_lag_rook(gpkg, X_test, lst, \"LagRook\")\n",
    "X_val_rook = spatial_lag_rook(gpkg, X_val, lst, \"LagRook\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "49caa834-6eab-4c7c-831f-e3dee67143a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_distance_band10000 = spatial_lag_distance_band(gpkg, X_train_split, lst, \"LagDistanceBand10000\", 10000)\n",
    "X_test_distance_band10000 = spatial_lag_distance_band(gpkg, X_test, lst, \"LagDistanceBand10000\", 10000)\n",
    "X_val_distance_band10000 = spatial_lag_distance_band(gpkg, X_val, lst, \"LagDistanceBand10000\", 10000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7ffc6b51-9171-47ed-a35a-066f8b867e99",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://onlinelibrary-wiley-com.tilburguniversity.idm.oclc.org/doi/10.1111/tgis.12729\n",
    "geo_X_train_rook = X_train_rook[[\"geometry\"]]\n",
    "geo_X_test_rook = X_test_rook[[\"geometry\"]]\n",
    "geo_X_val_rook= X_val_rook[[\"geometry\"]]\n",
    "\n",
    "geo_X_train_KNN8  = X_train_KNN8[[\"geometry\"]]\n",
    "geo_X_test_KNN8 = X_test_KNN8[[\"geometry\"]]\n",
    "geo_X_val_KNN8 = X_val_KNN8[[\"geometry\"]]\n",
    "\n",
    "\n",
    "geo_X_train_distance_band10000 = X_train_distance_band10000[[\"geometry\"]]\n",
    "geo_X_test_distance_band10000 = X_test_distance_band10000[[\"geometry\"]]\n",
    "geo_X_val_distance_band10000 = X_val_distance_band10000[[\"geometry\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "f9949336-05d1-4286-841a-bce7f769413a",
   "metadata": {},
   "outputs": [],
   "source": [
    "geo_X_train_rook.to_csv(\"geo_X_train_rook.csv\", index=False)\n",
    "geo_X_test_rook.to_csv(\"geo_X_test_rook.csv\", index=False)\n",
    "geo_X_val_rook.to_csv(\"geo_X_val_rook.csv\", index=False)\n",
    "\n",
    "geo_X_train_KNN8.to_csv(\"geo_X_train_KNN8.csv\", index=False)\n",
    "geo_X_test_KNN8.to_csv(\"geo_X_test_KNN8.csv\", index=False)\n",
    "geo_X_val_KNN8.to_csv(\"geo_X_val_KNN8.csv\", index=False)\n",
    "\n",
    "geo_X_train_distance_band10000.to_csv(\"geo_X_train_distance_band10000.csv\", index=False)\n",
    "geo_X_train_distance_band10000.to_csv(\"geo_X_train_distance_band10000.csv\", index=False)\n",
    "geo_X_train_distance_band10000.to_csv(\"geo_X_train_distance_band10000.csv\", index=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6e49fe9-5f71-4cf4-ac10-b5b60c246fc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop features not needed \n",
    "for features in [\"UniqueIdentifier\", \"HomePostalCode\", 'Residential Municipality',\"geometry\", \"ResidentialProvince\", \n",
    "                ]: \n",
    "    X_train_distance_band10000.drop([features], inplace=True, axis=1)\n",
    "    X_train_rook.drop([features], inplace=True, axis=1)\n",
    "    X_train_KNN8.drop([features], inplace=True, axis=1)\n",
    "    \n",
    "    X_val_distance_band10000.drop([features], inplace=True, axis=1)\n",
    "    X_val_rook.drop([features], inplace=True, axis=1)\n",
    "    X_val_KNN8.drop([features], inplace=True, axis=1)\n",
    "    \n",
    "    X_test_distance_band10000.drop([features], inplace=True, axis=1)\n",
    "    X_test_rook.drop([features], inplace=True, axis=1)\n",
    "    X_test_KNN8.drop([features], inplace=True, axis=1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2856284",
   "metadata": {},
   "source": [
    "<a id='section3'></a>\n",
    "# Encoding categorical features\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6e94b1e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "from sklearn.preprocessing import RobustScaler, OneHotEncoder, StandardScaler\n",
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "\n",
    "X_train_KNN8 , X_val_KNN8, X_test_KNN8= encoding_categorical_features(X_train_KNN8 , X_val_KNN8, X_test_KNN8)\n",
    "X_train_rook, X_val_rook, X_test_rook = encoding_categorical_features(X_train_rook, X_val_rook, X_test_rook)\n",
    "X_train_distance_band10000, X_val_distance_band10000, X_test_distance_band10000 = encoding_categorical_features(X_train_distance_band10000, X_val_distance_band10000, X_test_distance_band10000)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f3a41a8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ordinal_features = [\"AgeClass\",\"HouseholdSize\", \"DisposableIncome\",\"FrequencyOfUseOfNonEelectricBicycle\", \"FrequencyOfWalkingOutdoors\",\n",
    "                    \"NumberOfCarsInHousehold\",  \"EducationLevel\",\"FrequencyOfUseCcarAsAPassenger\"]\n",
    "numeric_features = [ \"TotalTravelDuration\", \"DepartureTime\",\"NumberOfTripLegs\"\n",
    "                    , \"NumBusStops\", 'RoadNetworkDensity','AvgDistToTrainStation',\"TripLegDistance\",\"N.OFHouseholdMemberUnd6\",\n",
    "    'AvgDistToSupermarket', 'NofBusinessEstablishments', 'SurroundingAddressDensity', \"LatitudeLiving\", \"LongitudeLiving\"]\n",
    "categorical_features = [\"TripPurpose\",\"WorkCommuteReimbursement\", ]\n",
    "binary_features = [\"Gender\",\"License\", \"Weekday\", \"ElectriBicycleIHousehold\"]\n",
    "pca_features = [\"PCA_dr_wl_cl_duration_2\",\"PCA_dr_wl_cl_duration_1\", \"PCA_dr_wl_cl_duration_3\"]\n",
    "\n",
    "\n",
    "lag_knn8 = [\"SurroundingAddressDensity_LagKNN8\",  \"AvgDistToTrainStation_LagKNN8\",   \"NumBusStops_LagKNN8\",\n",
    "    \"RoadNetworkDensity_LagKNN8\",   \"AvgDistToSupermarket_LagKNN8\",  \"NofBusinessEstablishments_LagKNN8\"]\n",
    "\n",
    "lag_dist10000= ['SurroundingAddressDensity_LagDistanceBand10000', 'AvgDistToTrainStation_LagDistanceBand10000',\n",
    "    'NumBusStops_LagDistanceBand10000',   'RoadNetworkDensity_LagDistanceBand10000',\n",
    "    'AvgDistToSupermarket_LagDistanceBand10000',   'NofBusinessEstablishments_LagDistanceBand10000']\n",
    "\n",
    "\n",
    "lag_rook = [\"SurroundingAddressDensity_LagRook\", \"AvgDistToTrainStation_LagRook\", \"NumBusStops_LagRook\",\n",
    "    \"RoadNetworkDensity_LagRook\",  \"AvgDistToSupermarket_LagRook\",   \"NofBusinessEstablishments_LagRook\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0e463843-8696-448a-a677-c7c51d7c2d0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaling features\n",
    "# X_train_split, X_val, X_test, ordinal_features, numeric_features, categorical_features, binary_features, lag_features, pca_features\n",
    "\n",
    "X_train_KNN8 , X_val_KNN8, X_test_KNN8, all_feature_namesKNN8, scaler = scaling_features(X_train_KNN8 , X_val_KNN8, X_test_KNN8, \n",
    "                                            ordinal_features, numeric_features, categorical_features, binary_features, lag_knn8, pca_features)\n",
    "\n",
    "X_train_rook, X_val_rook, X_test_rook, all_feature_namesrook, scaler = scaling_features(X_train_rook, X_val_rook, X_test_rook, \n",
    "                                            ordinal_features, numeric_features, categorical_features, binary_features, lag_rook, pca_features)\n",
    "\n",
    "X_train_distance_band10000, X_val_distance_band10000, X_test_distance_band10000, all_feature_dist10000, scaler= scaling_features(X_train_distance_band10000,\n",
    "                                              X_val_distance_band10000, X_test_distance_band10000, \n",
    "                                            ordinal_features, numeric_features, categorical_features, binary_features, lag_dist10000, pca_features )\n",
    "                                                                              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7b09010b-0031-4fb1-8fbe-da8a7e0d0a75",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_feature_dist10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "26892d8c-75d8-4d82-a84a-eb6232e5fba9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the y_data (encoded)\n",
    "y_train_split = np.loadtxt('y_train_split_encoded.txt', dtype=int)\n",
    "y_val = np.loadtxt('y_val_encoded.txt', dtype=int)\n",
    "y_test = np.loadtxt('y_test_encoded.txt', dtype=int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5252bf79-3b02-4fd5-8bc6-5e0c55b0d9e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# rescale features for explainable ai\n",
    "X_train_unscaled_numeric = scaler.inverse_transform(X_train_distance_band10000[:, :len(numeric_features)])  \n",
    "X_val_unscaled_numeric = scaler.inverse_transform(X_val_distance_band10000[:, :len(numeric_features)])\n",
    "X_test_unscaled_numeric = scaler.inverse_transform(X_test_distance_band10000[:, :len(numeric_features)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "92b9ac6d-e266-438d-bb28-4ecf5cd706fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# unscaled X_rook for shap analysis \n",
    "len_unscaled_numeric = len(numeric_features)\n",
    "\n",
    "# unscaled X_rook for shap analysis \n",
    "len_unscaled_numeric = len(numeric_features)\n",
    "\n",
    "\n",
    "\n",
    "X_test_dist10000_un = np.concatenate([\n",
    "    X_test_unscaled_numeric,\n",
    "    X_test_distance_band10000[:, len_unscaled_numeric:],  \n",
    "], axis=1)\n",
    "\n",
    "X_train_dist10000_un = np.concatenate([\n",
    "    X_train_unscaled_numeric,\n",
    "    X_train_distance_band10000[:, len_unscaled_numeric:],\n",
    "], axis=1)\n",
    "\n",
    "X_val_dist10000_un = np.concatenate([\n",
    "    X_val_unscaled_numeric,\n",
    "    X_val_distance_band10000[:, len_unscaled_numeric:],  \n",
    "], axis=1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "91ab4b59-310e-44ce-a445-49ec74ff49fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# best model\n",
    "np.save(\"X_train_distance_band10000\", X_train_distance_band10000)\n",
    "np.save(\"X_test_distance_band10000\", X_test_distance_band10000)\n",
    "np.save(\"X_val_distance_band10000\",X_val_distance_band10000)\n",
    "\n",
    "np.save(\"all_feature_dist10000\", all_feature_dist10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "db1231f5-b473-43a6-ae8f-396f4218b42e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save unscaled features\n",
    "np.save(\"X_train_dist10000_unn\", X_train_dist10000_un)\n",
    "np.save(\"X_val_dist10000_un\", X_val_dist10000_un)\n",
    "np.save(\"X_test_dist10000_un\", X_test_dist10000_un)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f331f669-1ec0-4d1c-abbd-ec4505ad3135",
   "metadata": {},
   "source": [
    "<a id='section4'></a>\n",
    "# KNN\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f6a7f03c-1406-4c06-88cd-ebd7c691b195",
   "metadata": {},
   "outputs": [],
   "source": [
    "import optuna\n",
    "from optuna import Trial\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.metrics import f1_score, accuracy_score\n",
    "random_state = 0\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from optuna.pruners import SuccessiveHalvingPruner\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "import torch\n",
    "from sklearn.metrics import f1_score\n",
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "\n",
    "from functions_spatialmodels import Objective_knn, Objective_rf, Objective_xgb, Objective_tabnet, all_metrics, predict, conf_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f96699b-a047-4ad1-8657-5770ed1e8766",
   "metadata": {},
   "source": [
    "# model:KNN_KNN8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "35e9522f-a3b7-4eb5-bccf-e9633fc78c8b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-28 19:52:42,272] A new study created in memory with name: no-name-8a1e2d7e-1bd9-4f7b-a5ef-4bbb71a85d15\n",
      "[I 2024-04-28 19:52:47,554] Trial 0 finished with value: 0.5478816207638229 and parameters: {'n_neighbors': 281, 'weights': 'uniform', 'metric': 'euclidean', 'algorithm': 'auto'}. Best is trial 0 with value: 0.5478816207638229.\n",
      "[I 2024-04-28 19:56:28,976] Trial 1 finished with value: 0.5461956828022773 and parameters: {'n_neighbors': 293, 'weights': 'uniform', 'metric': 'euclidean', 'algorithm': 'kd_tree'}. Best is trial 0 with value: 0.5478816207638229.\n",
      "[I 2024-04-28 20:00:04,549] Trial 2 finished with value: 0.5553697980109764 and parameters: {'n_neighbors': 259, 'weights': 'distance', 'metric': 'minkowski', 'algorithm': 'kd_tree'}. Best is trial 2 with value: 0.5553697980109764.\n",
      "[I 2024-04-28 20:05:25,394] Trial 3 finished with value: 0.5970512296692192 and parameters: {'n_neighbors': 252, 'weights': 'uniform', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:15:03,103] Trial 4 finished with value: 0.5946888083860675 and parameters: {'n_neighbors': 285, 'weights': 'uniform', 'metric': 'manhattan', 'algorithm': 'kd_tree'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:15:15,508] Trial 5 finished with value: 0.5556604269153509 and parameters: {'n_neighbors': 257, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'auto'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:15:32,906] Trial 6 finished with value: 0.5454305405336922 and parameters: {'n_neighbors': 301, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'brute'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:15:45,987] Trial 7 finished with value: 0.5537500691668362 and parameters: {'n_neighbors': 268, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'auto'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:29:31,047] Trial 8 finished with value: 0.5486916750975425 and parameters: {'n_neighbors': 271, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'ball_tree'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:39:38,181] Trial 9 finished with value: 0.5478816207638229 and parameters: {'n_neighbors': 281, 'weights': 'uniform', 'metric': 'euclidean', 'algorithm': 'kd_tree'}. Best is trial 3 with value: 0.5970512296692192.\n",
      "[I 2024-04-28 20:55:09,887] Trial 10 finished with value: 0.6004400047039853 and parameters: {'n_neighbors': 251, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:10:45,625] Trial 11 finished with value: 0.5999332190875573 and parameters: {'n_neighbors': 254, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:23:06,657] Trial 12 finished with value: 0.6004400047039853 and parameters: {'n_neighbors': 251, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:31:30,042] Trial 13 finished with value: 0.5991824730966866 and parameters: {'n_neighbors': 266, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:40:08,488] Trial 14 finished with value: 0.5955527521586712 and parameters: {'n_neighbors': 312, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:40:30,593] Trial 15 finished with value: 0.6002750089584639 and parameters: {'n_neighbors': 250, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'brute'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:48:18,725] Trial 16 finished with value: 0.5992633479445784 and parameters: {'n_neighbors': 264, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 21:56:30,928] Trial 17 finished with value: 0.5979562226084696 and parameters: {'n_neighbors': 275, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 22:04:03,126] Trial 18 finished with value: 0.5988243899844026 and parameters: {'n_neighbors': 262, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 10 with value: 0.6004400047039853.\n",
      "[I 2024-04-28 22:04:20,493] Trial 19 finished with value: 0.5979460177339877 and parameters: {'n_neighbors': 274, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'brute'}. Best is trial 10 with value: 0.6004400047039853.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.6004400047039853\n",
      "Best Params: \n",
      "  n_neighbors: 251\n",
      "  weights: distance\n",
      "  metric: manhattan\n",
      "  algorithm: ball_tree\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "import joblib\n",
    "\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_knn(trial, X_train_KNN8, y_train_split, X_val_KNN8, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_knn_KNN8.pkl')\n",
    "\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6de31ec9",
   "metadata": {},
   "source": [
    "# model:RF_KNN8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "bba00f24",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 17:44:34,230] A new study created in memory with name: no-name-20c76ac7-f6a3-4f26-9dce-b02f47fcf4b0\n",
      "[I 2024-04-29 17:44:34,308] Trial 0 finished with value: 0.5125151899321583 and parameters: {'n_estimators': 1, 'max_depth': 4, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 0 with value: 0.5125151899321583.\n",
      "[I 2024-04-29 17:44:34,942] Trial 1 finished with value: 0.6945437358687714 and parameters: {'n_estimators': 2, 'max_depth': 22, 'min_samples_leaf': 9, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 1 with value: 0.6945437358687714.\n",
      "[I 2024-04-29 17:44:36,254] Trial 2 finished with value: 0.6999306622527346 and parameters: {'n_estimators': 1, 'max_depth': 15, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': False, 'criterion': 'gini'}. Best is trial 2 with value: 0.6999306622527346.\n",
      "[I 2024-04-29 17:44:47,832] Trial 3 finished with value: 0.6962427978219745 and parameters: {'n_estimators': 8, 'max_depth': 29, 'min_samples_leaf': 7, 'max_features': None, 'bootstrap': False, 'criterion': 'gini'}. Best is trial 2 with value: 0.6999306622527346.\n",
      "[I 2024-04-29 17:44:48,191] Trial 4 finished with value: 0.6336313239700297 and parameters: {'n_estimators': 1, 'max_depth': 22, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 2 with value: 0.6999306622527346.\n",
      "[I 2024-04-29 17:44:48,301] Trial 5 finished with value: 0.5872064691659541 and parameters: {'n_estimators': 1, 'max_depth': 6, 'min_samples_leaf': 7, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 2 with value: 0.6999306622527346.\n",
      "[I 2024-04-29 17:46:03,673] Trial 6 finished with value: 0.7578222902044619 and parameters: {'n_estimators': 58, 'max_depth': 18, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 6 with value: 0.7578222902044619.\n",
      "[I 2024-04-29 17:46:08,623] Trial 7 finished with value: 0.7335398063776293 and parameters: {'n_estimators': 24, 'max_depth': 19, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 6 with value: 0.7578222902044619.\n",
      "[I 2024-04-29 17:46:08,733] Trial 8 finished with value: 0.542269141001726 and parameters: {'n_estimators': 1, 'max_depth': 6, 'min_samples_leaf': 9, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 6 with value: 0.7578222902044619.\n",
      "[I 2024-04-29 17:46:11,271] Trial 9 finished with value: 0.7347290573646654 and parameters: {'n_estimators': 10, 'max_depth': 28, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 6 with value: 0.7578222902044619.\n",
      "[I 2024-04-29 17:52:14,600] Trial 10 finished with value: 0.7583047439478844 and parameters: {'n_estimators': 278, 'max_depth': 40, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 10 with value: 0.7583047439478844.\n",
      "[I 2024-04-29 17:58:38,079] Trial 11 finished with value: 0.7584561075898516 and parameters: {'n_estimators': 286, 'max_depth': 40, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:03:53,189] Trial 12 finished with value: 0.7582397452851725 and parameters: {'n_estimators': 235, 'max_depth': 40, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:10:09,764] Trial 13 finished with value: 0.758234574932884 and parameters: {'n_estimators': 282, 'max_depth': 40, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:10:29,256] Trial 14 finished with value: 0.7242997936341331 and parameters: {'n_estimators': 98, 'max_depth': 34, 'min_samples_leaf': 9, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:13:02,554] Trial 15 finished with value: 0.7571376041845694 and parameters: {'n_estimators': 115, 'max_depth': 34, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:13:47,204] Trial 16 finished with value: 0.7573283112250702 and parameters: {'n_estimators': 34, 'max_depth': 35, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:16:50,200] Trial 17 finished with value: 0.757749408162607 and parameters: {'n_estimators': 138, 'max_depth': 27, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:16:51,170] Trial 18 finished with value: 0.6988539172772924 and parameters: {'n_estimators': 5, 'max_depth': 37, 'min_samples_leaf': 9, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n",
      "[I 2024-04-29 18:18:01,958] Trial 19 finished with value: 0.7553251542934059 and parameters: {'n_estimators': 50, 'max_depth': 31, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 11 with value: 0.7584561075898516.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7584561075898516\n",
      "Best Params: \n",
      "  n_estimators: 286\n",
      "  max_depth: 40\n",
      "  min_samples_leaf: 9\n",
      "  max_features: None\n",
      "  bootstrap: True\n",
      "  criterion: entropy\n"
     ]
    }
   ],
   "source": [
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_rf(trial, X_train_KNN8, y_train_split, X_val_KNN8, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_rf_KNN8.pkl')\n",
    "\n",
    "\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7e1845d",
   "metadata": {},
   "source": [
    "https://link-springer-com.tilburguniversity.idm.oclc.org/article/10.1007/s11042-024-18426-2#Tab6"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78fcf66f",
   "metadata": {},
   "source": [
    " # model:XGB_KNN8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d87da217",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-28 18:10:09,299] A new study created in memory with name: no-name-b8471e22-67e6-4ed0-8cb1-22236568560f\n",
      "[I 2024-04-28 18:10:14,317] Trial 0 finished with value: 0.7576270769119007 and parameters: {'learning_rate': 0.036434821689901746, 'n_estimators': 205, 'max_depth': 24, 'min_child_weight': 193.70427918411966, 'gamma': 0.2870772380606825, 'subsample': 0.735493896251469, 'colsample_bytree': 0.5001810418007931, 'reg_alpha': 28.702170564734992, 'reg_lambda': 10.54488923868155}. Best is trial 0 with value: 0.7576270769119007.\n",
      "[I 2024-04-28 18:10:33,936] Trial 1 finished with value: 0.6982318774477323 and parameters: {'learning_rate': 0.010083991263950496, 'n_estimators': 806, 'max_depth': 15, 'min_child_weight': 174.37664112237545, 'gamma': 0.9597607483817739, 'subsample': 0.6467693487419408, 'colsample_bytree': 0.15113682612229346, 'reg_alpha': 27.37307409262182, 'reg_lambda': 4.201995844101445}. Best is trial 0 with value: 0.7576270769119007.\n",
      "[I 2024-04-28 18:10:51,228] Trial 2 finished with value: 0.7806387105244316 and parameters: {'learning_rate': 0.027428762729185555, 'n_estimators': 453, 'max_depth': 8, 'min_child_weight': 15.197390337695358, 'gamma': 0.1541709092882836, 'subsample': 0.9456515175941944, 'colsample_bytree': 0.7977514767165595, 'reg_alpha': 8.118316631641632, 'reg_lambda': 21.477998621047682}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:11:08,880] Trial 3 finished with value: 0.722342405771969 and parameters: {'learning_rate': 0.01822536895986118, 'n_estimators': 555, 'max_depth': 14, 'min_child_weight': 199.56176152932898, 'gamma': 0.5670919075577237, 'subsample': 0.6036890668681429, 'colsample_bytree': 0.17823550700234556, 'reg_alpha': 13.489612664717882, 'reg_lambda': 15.48575669935812}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:11:20,008] Trial 4 finished with value: 0.7751169205144844 and parameters: {'learning_rate': 0.046276262347710394, 'n_estimators': 267, 'max_depth': 14, 'min_child_weight': 65.27033190535543, 'gamma': 0.22636454487529, 'subsample': 0.7631835400564899, 'colsample_bytree': 0.8352413005452658, 'reg_alpha': 25.86218797357814, 'reg_lambda': 7.80435601462157}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:11:37,258] Trial 5 finished with value: 0.7701206433102976 and parameters: {'learning_rate': 0.4359959887554596, 'n_estimators': 804, 'max_depth': 3, 'min_child_weight': 144.44028707858828, 'gamma': 0.3940096565205, 'subsample': 0.5934561687101196, 'colsample_bytree': 0.23278763234681824, 'reg_alpha': 9.139053780989988, 'reg_lambda': 27.471799180748175}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:12:13,247] Trial 6 finished with value: 0.7759048231992731 and parameters: {'learning_rate': 0.015687325351113203, 'n_estimators': 966, 'max_depth': 26, 'min_child_weight': 84.76150732562331, 'gamma': 0.1492570973647428, 'subsample': 0.642203141418707, 'colsample_bytree': 0.7127552133707593, 'reg_alpha': 23.528977938282452, 'reg_lambda': 5.743318231559984}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:12:25,778] Trial 7 finished with value: 0.7740952352265428 and parameters: {'learning_rate': 0.12417733292364594, 'n_estimators': 471, 'max_depth': 13, 'min_child_weight': 58.46886754382522, 'gamma': 0.9927370498432845, 'subsample': 0.552314070896553, 'colsample_bytree': 0.3096364276390821, 'reg_alpha': 22.121214877180968, 'reg_lambda': 10.893674483199367}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:12:52,417] Trial 8 finished with value: 0.7696062278082337 and parameters: {'learning_rate': 0.049776869883810027, 'n_estimators': 864, 'max_depth': 7, 'min_child_weight': 123.47166150160938, 'gamma': 0.13586336356862405, 'subsample': 0.5236241704118383, 'colsample_bytree': 0.23809427171282466, 'reg_alpha': 23.08507364599363, 'reg_lambda': 3.4659834121433164}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:12:57,040] Trial 9 finished with value: 0.767351577662818 and parameters: {'learning_rate': 0.9554167174774308, 'n_estimators': 133, 'max_depth': 29, 'min_child_weight': 43.53702232072716, 'gamma': 0.2180277662685599, 'subsample': 0.632678097038762, 'colsample_bytree': 0.5142827317017873, 'reg_alpha': 28.974773032691633, 'reg_lambda': 16.01669077152718}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:13:10,170] Trial 10 finished with value: 0.778653980441472 and parameters: {'learning_rate': 0.17324035609286165, 'n_estimators': 525, 'max_depth': 8, 'min_child_weight': 5.389959923674132, 'gamma': 0.6263760182149074, 'subsample': 0.9854447490065823, 'colsample_bytree': 0.8938870238199171, 'reg_alpha': 0.637627604272482, 'reg_lambda': 23.517239175523162}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:13:22,931] Trial 11 finished with value: 0.7774174704240165 and parameters: {'learning_rate': 0.16362632552572354, 'n_estimators': 552, 'max_depth': 8, 'min_child_weight': 0.23462216824671067, 'gamma': 0.6526989327769659, 'subsample': 0.97866942368487, 'colsample_bytree': 0.9414887437275024, 'reg_alpha': 1.434760779187983, 'reg_lambda': 23.429140240758755}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:13:31,121] Trial 12 finished with value: 0.7776025273280032 and parameters: {'learning_rate': 0.25979882035294405, 'n_estimators': 382, 'max_depth': 8, 'min_child_weight': 3.581857043920632, 'gamma': 0.7633460398380583, 'subsample': 0.9936661957773262, 'colsample_bytree': 0.7113234217349479, 'reg_alpha': 0.6406342299461866, 'reg_lambda': 21.560488560334196}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:13:59,535] Trial 13 finished with value: 0.7779960790429615 and parameters: {'learning_rate': 0.07587099867759668, 'n_estimators': 686, 'max_depth': 19, 'min_child_weight': 29.069846452883567, 'gamma': 0.42275326120130974, 'subsample': 0.8911212954788669, 'colsample_bytree': 0.9335459635489273, 'reg_alpha': 7.392063674817661, 'reg_lambda': 29.63237408675731}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:14:08,627] Trial 14 finished with value: 0.7764824286084642 and parameters: {'learning_rate': 0.2657576862278029, 'n_estimators': 394, 'max_depth': 3, 'min_child_weight': 28.0782604381981, 'gamma': 0.7806811980590523, 'subsample': 0.8874337494264124, 'colsample_bytree': 0.7198431018543313, 'reg_alpha': 6.295893892755422, 'reg_lambda': 20.54738063325802}. Best is trial 2 with value: 0.7806387105244316.\n",
      "[I 2024-04-28 18:14:35,081] Trial 15 finished with value: 0.7809292916040592 and parameters: {'learning_rate': 0.024279061436036486, 'n_estimators': 649, 'max_depth': 10, 'min_child_weight': 99.10156224588998, 'gamma': 0.04802592177622489, 'subsample': 0.9044671934206918, 'colsample_bytree': 0.8227773761892602, 'reg_alpha': 4.273779069973355, 'reg_lambda': 25.467110647348857}. Best is trial 15 with value: 0.7809292916040592.\n",
      "[I 2024-04-28 18:15:02,744] Trial 16 finished with value: 0.7786103658373444 and parameters: {'learning_rate': 0.023291240821693683, 'n_estimators': 662, 'max_depth': 20, 'min_child_weight': 105.6718849160192, 'gamma': 0.008408020889124512, 'subsample': 0.887400282235358, 'colsample_bytree': 0.6284711430592441, 'reg_alpha': 12.960126406460926, 'reg_lambda': 18.027728266127045}. Best is trial 15 with value: 0.7809292916040592.\n",
      "[I 2024-04-28 18:15:29,344] Trial 17 finished with value: 0.7806951710368457 and parameters: {'learning_rate': 0.029357976783648515, 'n_estimators': 698, 'max_depth': 11, 'min_child_weight': 144.21530068219982, 'gamma': 0.013466628189910596, 'subsample': 0.8158827580537931, 'colsample_bytree': 0.8116292404841284, 'reg_alpha': 4.689025353759652, 'reg_lambda': 26.11277190915048}. Best is trial 15 with value: 0.7809292916040592.\n",
      "[I 2024-04-28 18:15:55,961] Trial 18 finished with value: 0.7785930126357554 and parameters: {'learning_rate': 0.07413553649457885, 'n_estimators': 710, 'max_depth': 11, 'min_child_weight': 141.47573688936336, 'gamma': 0.021846161683568927, 'subsample': 0.7891215241357744, 'colsample_bytree': 0.9959915060065645, 'reg_alpha': 18.592620345874543, 'reg_lambda': 26.800290523542017}. Best is trial 15 with value: 0.7809292916040592.\n",
      "[I 2024-04-28 18:16:21,693] Trial 19 finished with value: 0.7617099536608998 and parameters: {'learning_rate': 0.011154819648018312, 'n_estimators': 626, 'max_depth': 18, 'min_child_weight': 96.1220882944574, 'gamma': 0.3598182068540755, 'subsample': 0.825947289707054, 'colsample_bytree': 0.4329552874009516, 'reg_alpha': 4.274940008290443, 'reg_lambda': 25.949752408842983}. Best is trial 15 with value: 0.7809292916040592.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7809292916040592\n",
      "Best Params: \n",
      "  learning_rate: 0.024279061436036486\n",
      "  n_estimators: 649\n",
      "  max_depth: 10\n",
      "  min_child_weight: 99.10156224588998\n",
      "  gamma: 0.04802592177622489\n",
      "  subsample: 0.9044671934206918\n",
      "  colsample_bytree: 0.8227773761892602\n",
      "  reg_alpha: 4.273779069973355\n",
      "  reg_lambda: 25.467110647348857\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_xgb(trial, X_train_KNN8, y_train_split, X_val_KNN8, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_xgb_KNN8.pkl')\n",
    "\n",
    "# print the best scores for each trial \n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4a1d6d6",
   "metadata": {},
   "source": [
    " # model:Tabnet_KNN8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fb351b5a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 11:35:21,722] A new study created in memory with name: no-name-f66ea386-f56b-4586-937c-f0cf3ea6b59f\n",
      "[I 2024-04-29 11:37:03,717] Trial 0 finished with value: 0.7408276043109414 and parameters: {'mask_type': 'sparsemax', 'n_steps': 5, 'n_d': 12, 'gamma': 1.4, 'n_shared': 2, 'lambda_sparse': 6.7803271234852655e-06, 'patienceScheduler': 4}. Best is trial 0 with value: 0.7408276043109414.\n",
      "[I 2024-04-29 11:40:25,833] Trial 1 finished with value: 0.6319713520768702 and parameters: {'mask_type': 'entmax', 'n_steps': 14, 'n_d': 8, 'gamma': 1.2, 'n_shared': 4, 'lambda_sparse': 1.5597727093733072e-06, 'patienceScheduler': 8}. Best is trial 0 with value: 0.7408276043109414.\n",
      "[I 2024-04-29 11:42:27,377] Trial 2 finished with value: 0.7630653792034547 and parameters: {'mask_type': 'entmax', 'n_steps': 9, 'n_d': 4, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 9.186805534575696e-05, 'patienceScheduler': 4}. Best is trial 2 with value: 0.7630653792034547.\n",
      "[I 2024-04-29 11:45:27,741] Trial 3 finished with value: 0.7038899368735331 and parameters: {'mask_type': 'sparsemax', 'n_steps': 14, 'n_d': 10, 'gamma': 1.2, 'n_shared': 4, 'lambda_sparse': 0.00016986005926419767, 'patienceScheduler': 4}. Best is trial 2 with value: 0.7630653792034547.\n",
      "[I 2024-04-29 11:47:10,458] Trial 4 finished with value: 0.7444739426755322 and parameters: {'mask_type': 'sparsemax', 'n_steps': 6, 'n_d': 8, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 5.85700731521651e-06, 'patienceScheduler': 5}. Best is trial 2 with value: 0.7630653792034547.\n",
      "[I 2024-04-29 11:49:42,394] Trial 5 finished with value: 0.76638113382244 and parameters: {'mask_type': 'entmax', 'n_steps': 13, 'n_d': 8, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 2.450715921956413e-06, 'patienceScheduler': 3}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 11:51:25,625] Trial 6 finished with value: 0.7499608036705357 and parameters: {'mask_type': 'entmax', 'n_steps': 4, 'n_d': 6, 'gamma': 1.2, 'n_shared': 4, 'lambda_sparse': 0.0004290783731388962, 'patienceScheduler': 8}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 11:54:08,060] Trial 7 finished with value: 0.6721421424727125 and parameters: {'mask_type': 'entmax', 'n_steps': 14, 'n_d': 4, 'gamma': 1.2, 'n_shared': 2, 'lambda_sparse': 6.503939855355825e-06, 'patienceScheduler': 3}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 11:55:53,988] Trial 8 finished with value: 0.7254327223381166 and parameters: {'mask_type': 'sparsemax', 'n_steps': 4, 'n_d': 8, 'gamma': 1.6, 'n_shared': 4, 'lambda_sparse': 0.0005493137115995553, 'patienceScheduler': 10}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 11:57:25,379] Trial 9 finished with value: 0.7603498626837678 and parameters: {'mask_type': 'entmax', 'n_steps': 4, 'n_d': 8, 'gamma': 1.6, 'n_shared': 1, 'lambda_sparse': 0.00018999425296726917, 'patienceScheduler': 8}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 11:59:56,681] Trial 10 finished with value: 0.6943076499846006 and parameters: {'mask_type': 'entmax', 'n_steps': 11, 'n_d': 2, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 1.277869405485923e-06, 'patienceScheduler': 6}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:01:52,553] Trial 11 finished with value: 0.7561976863995366 and parameters: {'mask_type': 'entmax', 'n_steps': 9, 'n_d': 4, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 4.5644115300099104e-05, 'patienceScheduler': 3}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:04:06,834] Trial 12 finished with value: 0.7521857031823601 and parameters: {'mask_type': 'entmax', 'n_steps': 10, 'n_d': 4, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 3.324602842996849e-05, 'patienceScheduler': 5}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:06:17,164] Trial 13 finished with value: 0.7297776534982214 and parameters: {'mask_type': 'entmax', 'n_steps': 8, 'n_d': 6, 'gamma': 1.4, 'n_shared': 3, 'lambda_sparse': 8.88054552828414e-05, 'patienceScheduler': 3}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:08:30,325] Trial 14 finished with value: 0.6328928118355432 and parameters: {'mask_type': 'entmax', 'n_steps': 12, 'n_d': 2, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 2.0921866767305555e-05, 'patienceScheduler': 5}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:10:20,981] Trial 15 finished with value: 0.7647903243403404 and parameters: {'mask_type': 'entmax', 'n_steps': 7, 'n_d': 12, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 1.2313798913024803e-05, 'patienceScheduler': 4}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:12:20,984] Trial 16 finished with value: 0.7454700740573522 and parameters: {'mask_type': 'entmax', 'n_steps': 7, 'n_d': 12, 'gamma': 1.4, 'n_shared': 3, 'lambda_sparse': 2.7075166242938464e-06, 'patienceScheduler': 6}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:14:45,983] Trial 17 finished with value: 0.704497891067078 and parameters: {'mask_type': 'entmax', 'n_steps': 12, 'n_d': 10, 'gamma': 1.2, 'n_shared': 2, 'lambda_sparse': 1.694326527085282e-05, 'patienceScheduler': 7}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:16:40,986] Trial 18 finished with value: 0.7503373932510609 and parameters: {'mask_type': 'sparsemax', 'n_steps': 7, 'n_d': 10, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 2.9909395093697757e-06, 'patienceScheduler': 4}. Best is trial 5 with value: 0.76638113382244.\n",
      "[I 2024-04-29 12:19:43,614] Trial 19 finished with value: 0.6748216407132027 and parameters: {'mask_type': 'entmax', 'n_steps': 15, 'n_d': 12, 'gamma': 1.2, 'n_shared': 3, 'lambda_sparse': 1.1625619873886619e-05, 'patienceScheduler': 10}. Best is trial 5 with value: 0.76638113382244.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.76638113382244\n",
      "Best Params: \n",
      "  mask_type: entmax\n",
      "  n_steps: 13\n",
      "  n_d: 8\n",
      "  gamma: 1.0\n",
      "  n_shared: 2\n",
      "  lambda_sparse: 2.450715921956413e-06\n",
      "  patienceScheduler: 3\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_tabnet(trial, X_train_KNN8, y_train_split, X_val_KNN8, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_tabnet_KNN8.pkl')\n",
    "\n",
    "# Retrieve the best trial and print its score and parameters\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e9c9a079-04b4-4a19-b1a3-174c09af5d60",
   "metadata": {},
   "source": [
    "<a id='section5'></a>\n",
    "# Distanceband\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c2102b71-b2cf-46e2-8ddd-43dbc1027e7b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 08:13:46,105] A new study created in memory with name: no-name-6d6f16a7-a973-4f7b-abc9-b1c375f73a89\n",
      "[I 2024-04-29 08:13:58,850] Trial 0 finished with value: 0.5470929539182521 and parameters: {'n_neighbors': 284, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'brute'}. Best is trial 0 with value: 0.5470929539182521.\n",
      "[I 2024-04-29 08:27:23,747] Trial 1 finished with value: 0.5996537198062029 and parameters: {'n_neighbors': 257, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 08:36:46,163] Trial 2 finished with value: 0.5534647016820777 and parameters: {'n_neighbors': 272, 'weights': 'distance', 'metric': 'minkowski', 'algorithm': 'kd_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 08:49:35,067] Trial 3 finished with value: 0.5515930046471684 and parameters: {'n_neighbors': 290, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'ball_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 08:49:48,684] Trial 4 finished with value: 0.5474523046524555 and parameters: {'n_neighbors': 283, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'auto'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 09:02:38,722] Trial 5 finished with value: 0.549666571332043 and parameters: {'n_neighbors': 266, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'ball_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 09:11:17,233] Trial 6 finished with value: 0.5515930046471684 and parameters: {'n_neighbors': 290, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'kd_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 09:21:57,421] Trial 7 finished with value: 0.5508530606666977 and parameters: {'n_neighbors': 296, 'weights': 'distance', 'metric': 'minkowski', 'algorithm': 'ball_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 09:22:09,424] Trial 8 finished with value: 0.5461956828022773 and parameters: {'n_neighbors': 293, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'brute'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 09:34:49,432] Trial 9 finished with value: 0.5931955110669547 and parameters: {'n_neighbors': 312, 'weights': 'uniform', 'metric': 'manhattan', 'algorithm': 'kd_tree'}. Best is trial 1 with value: 0.5996537198062029.\n",
      "[I 2024-04-29 09:35:18,443] Trial 10 finished with value: 0.6002014904122926 and parameters: {'n_neighbors': 252, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 10 with value: 0.6002014904122926.\n",
      "[I 2024-04-29 09:35:35,906] Trial 11 finished with value: 0.6004400047039853 and parameters: {'n_neighbors': 251, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:35:53,941] Trial 12 finished with value: 0.6004400047039853 and parameters: {'n_neighbors': 251, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:36:11,791] Trial 13 finished with value: 0.5992633479445784 and parameters: {'n_neighbors': 264, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:36:29,091] Trial 14 finished with value: 0.6002750089584639 and parameters: {'n_neighbors': 250, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:36:46,779] Trial 15 finished with value: 0.5996823801510345 and parameters: {'n_neighbors': 260, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:37:04,054] Trial 16 finished with value: 0.5983112233800025 and parameters: {'n_neighbors': 272, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:37:21,283] Trial 17 finished with value: 0.5983802958208083 and parameters: {'n_neighbors': 273, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:37:26,770] Trial 18 finished with value: 0.5563153146381157 and parameters: {'n_neighbors': 256, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'auto'}. Best is trial 11 with value: 0.6004400047039853.\n",
      "[I 2024-04-29 09:37:45,429] Trial 19 finished with value: 0.599471315329918 and parameters: {'n_neighbors': 265, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'brute'}. Best is trial 11 with value: 0.6004400047039853.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.6004400047039853\n",
      "Best Params: \n",
      "  n_neighbors: 251\n",
      "  weights: distance\n",
      "  metric: manhattan\n",
      "  algorithm: auto\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "import joblib\n",
    "\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_knn(trial, X_train_KNN8, y_train_split, X_val_KNN8, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_knn_dist10000.pkl')\n",
    "\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91b4978b-85c5-4cd4-bda3-80a7a8e9924a",
   "metadata": {},
   "source": [
    "# model:Rf_distanceband10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "544e2e4e-6b55-418b-8dd9-8aefc4e03323",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 17:07:46,861] A new study created in memory with name: no-name-8f3d2571-fcac-4f85-ae3d-bf5b8f2d52de\n",
      "[I 2024-04-29 17:07:50,449] Trial 0 finished with value: 0.6969055191703453 and parameters: {'n_estimators': 5, 'max_depth': 35, 'min_samples_leaf': 9, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 0 with value: 0.6969055191703453.\n",
      "[I 2024-04-29 17:07:57,794] Trial 1 finished with value: 0.7067066821949253 and parameters: {'n_estimators': 9, 'max_depth': 14, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 1 with value: 0.7067066821949253.\n",
      "[I 2024-04-29 17:08:02,364] Trial 2 finished with value: 0.6969546225530013 and parameters: {'n_estimators': 4, 'max_depth': 29, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 1 with value: 0.7067066821949253.\n",
      "[I 2024-04-29 17:08:10,912] Trial 3 finished with value: 0.7209577297257566 and parameters: {'n_estimators': 7, 'max_depth': 39, 'min_samples_leaf': 9, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 3 with value: 0.7209577297257566.\n",
      "[I 2024-04-29 17:08:11,958] Trial 4 finished with value: 0.6109182521359743 and parameters: {'n_estimators': 1, 'max_depth': 40, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 3 with value: 0.7209577297257566.\n",
      "[I 2024-04-29 17:33:51,101] Trial 5 finished with value: 0.6876518266841667 and parameters: {'n_estimators': 293, 'max_depth': 36, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 3 with value: 0.7209577297257566.\n",
      "[I 2024-04-29 17:33:57,347] Trial 6 finished with value: 0.7314246443208847 and parameters: {'n_estimators': 24, 'max_depth': 35, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 6 with value: 0.7314246443208847.\n",
      "[I 2024-04-29 17:34:02,939] Trial 7 finished with value: 0.6972361793790995 and parameters: {'n_estimators': 4, 'max_depth': 35, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': False, 'criterion': 'gini'}. Best is trial 6 with value: 0.7314246443208847.\n",
      "[I 2024-04-29 17:34:10,801] Trial 8 finished with value: 0.5426741678705438 and parameters: {'n_estimators': 235, 'max_depth': 2, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 6 with value: 0.7314246443208847.\n",
      "[I 2024-04-29 17:34:18,321] Trial 9 finished with value: 0.6863632701262412 and parameters: {'n_estimators': 4, 'max_depth': 27, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 6 with value: 0.7314246443208847.\n",
      "[I 2024-04-29 17:34:25,574] Trial 10 finished with value: 0.7258876004184451 and parameters: {'n_estimators': 43, 'max_depth': 16, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 6 with value: 0.7314246443208847.\n",
      "[I 2024-04-29 17:34:34,294] Trial 11 finished with value: 0.7280916928403942 and parameters: {'n_estimators': 50, 'max_depth': 17, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 6 with value: 0.7314246443208847.\n",
      "[I 2024-04-29 17:34:40,974] Trial 12 finished with value: 0.7320091656563443 and parameters: {'n_estimators': 35, 'max_depth': 21, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 12 with value: 0.7320091656563443.\n",
      "[I 2024-04-29 17:34:47,112] Trial 13 finished with value: 0.7332308750961479 and parameters: {'n_estimators': 31, 'max_depth': 25, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 13 with value: 0.7332308750961479.\n",
      "[I 2024-04-29 17:35:08,111] Trial 14 finished with value: 0.7310895258315977 and parameters: {'n_estimators': 110, 'max_depth': 23, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 13 with value: 0.7332308750961479.\n",
      "[I 2024-04-29 17:35:09,732] Trial 15 finished with value: 0.6580089834235697 and parameters: {'n_estimators': 23, 'max_depth': 7, 'min_samples_leaf': 7, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 13 with value: 0.7332308750961479.\n",
      "[I 2024-04-29 17:35:24,971] Trial 16 finished with value: 0.7304450453471251 and parameters: {'n_estimators': 80, 'max_depth': 23, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 13 with value: 0.7332308750961479.\n",
      "[I 2024-04-29 17:35:27,250] Trial 17 finished with value: 0.7100938626826785 and parameters: {'n_estimators': 18, 'max_depth': 11, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 13 with value: 0.7332308750961479.\n",
      "[I 2024-04-29 17:35:27,499] Trial 18 finished with value: 0.6169494484209139 and parameters: {'n_estimators': 1, 'max_depth': 27, 'min_samples_leaf': 7, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 13 with value: 0.7332308750961479.\n",
      "[I 2024-04-29 17:37:55,518] Trial 19 finished with value: 0.7549852047856843 and parameters: {'n_estimators': 130, 'max_depth': 20, 'min_samples_leaf': 7, 'max_features': None, 'bootstrap': True, 'criterion': 'gini'}. Best is trial 19 with value: 0.7549852047856843.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7549852047856843\n",
      "Best Params: \n",
      "  n_estimators: 130\n",
      "  max_depth: 20\n",
      "  min_samples_leaf: 7\n",
      "  max_features: None\n",
      "  bootstrap: True\n",
      "  criterion: gini\n"
     ]
    }
   ],
   "source": [
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_rf(trial, X_train_distance_band10000, y_train_split, X_val_distance_band10000, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_rf_dist10000.pkl')\n",
    "\n",
    "\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a29b2083-12b4-4cb3-a696-2b493f96d142",
   "metadata": {},
   "source": [
    "# model:XGB_distanceband10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "6090db93-37d3-40a2-84a1-a599691e3532",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-28 18:43:39,522] A new study created in memory with name: no-name-bb5eb330-7a40-47fd-81e7-da292548b7ba\n",
      "[I 2024-04-28 18:43:54,627] Trial 0 finished with value: 0.7764835075183829 and parameters: {'learning_rate': 0.12380723307604931, 'n_estimators': 236, 'max_depth': 24, 'min_child_weight': 23.787238201078345, 'gamma': 0.15235831862651594, 'subsample': 0.9979435353100881, 'colsample_bytree': 0.9428768743862416, 'reg_alpha': 29.68634515505148, 'reg_lambda': 8.454673840202547}. Best is trial 0 with value: 0.7764835075183829.\n",
      "[I 2024-04-28 18:44:13,372] Trial 1 finished with value: 0.7761926327936353 and parameters: {'learning_rate': 0.13335153446393727, 'n_estimators': 462, 'max_depth': 6, 'min_child_weight': 199.1378221202431, 'gamma': 0.16878847911652084, 'subsample': 0.6954674992567818, 'colsample_bytree': 0.6904319638024732, 'reg_alpha': 18.10820595979309, 'reg_lambda': 26.241235797426175}. Best is trial 0 with value: 0.7764835075183829.\n",
      "[I 2024-04-28 18:44:36,632] Trial 2 finished with value: 0.7786999821188056 and parameters: {'learning_rate': 0.11749744345049716, 'n_estimators': 757, 'max_depth': 15, 'min_child_weight': 40.49860265840248, 'gamma': 0.6791157040541753, 'subsample': 0.955876025711774, 'colsample_bytree': 0.8361663958932465, 'reg_alpha': 3.2735042646961157, 'reg_lambda': 13.058089567326572}. Best is trial 2 with value: 0.7786999821188056.\n",
      "[I 2024-04-28 18:45:28,227] Trial 3 finished with value: 0.7777903620679397 and parameters: {'learning_rate': 0.034578843868105724, 'n_estimators': 750, 'max_depth': 27, 'min_child_weight': 175.0977868761837, 'gamma': 0.5218605958708789, 'subsample': 0.9660439791072015, 'colsample_bytree': 0.33771914938605163, 'reg_alpha': 1.0876084803643482, 'reg_lambda': 13.675975554463873}. Best is trial 2 with value: 0.7786999821188056.\n",
      "[I 2024-04-28 18:46:08,280] Trial 4 finished with value: 0.7789660106529739 and parameters: {'learning_rate': 0.061804002194512236, 'n_estimators': 596, 'max_depth': 13, 'min_child_weight': 63.504162203068574, 'gamma': 0.04221923199148658, 'subsample': 0.5994259314164685, 'colsample_bytree': 0.9418117539917159, 'reg_alpha': 9.752206003923074, 'reg_lambda': 29.7743944905979}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:46:35,829] Trial 5 finished with value: 0.7705302205885259 and parameters: {'learning_rate': 0.013955195775676754, 'n_estimators': 387, 'max_depth': 19, 'min_child_weight': 51.661337612292094, 'gamma': 0.6586499175127029, 'subsample': 0.7483054432273755, 'colsample_bytree': 0.8451856743122803, 'reg_alpha': 5.650429393952642, 'reg_lambda': 22.89198869812337}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:47:00,909] Trial 6 finished with value: 0.7725313421064625 and parameters: {'learning_rate': 0.09561679597715933, 'n_estimators': 717, 'max_depth': 4, 'min_child_weight': 53.7269696604586, 'gamma': 0.45924140859281737, 'subsample': 0.7967003802222311, 'colsample_bytree': 0.2082413936803617, 'reg_alpha': 5.76950190384769, 'reg_lambda': 11.70169325965854}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:47:31,354] Trial 7 finished with value: 0.768001856722221 and parameters: {'learning_rate': 0.02013914177005351, 'n_estimators': 474, 'max_depth': 18, 'min_child_weight': 158.43084552517965, 'gamma': 0.07236008639269897, 'subsample': 0.6094695773105058, 'colsample_bytree': 0.45155855130711275, 'reg_alpha': 1.2460522990105016, 'reg_lambda': 12.266295913897624}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:47:48,173] Trial 8 finished with value: 0.7619607987502534 and parameters: {'learning_rate': 0.17960405204988036, 'n_estimators': 461, 'max_depth': 11, 'min_child_weight': 181.29129233746713, 'gamma': 0.8476287897550131, 'subsample': 0.9931132882496841, 'colsample_bytree': 0.12122002568351854, 'reg_alpha': 20.919685473507066, 'reg_lambda': 1.2430984418519597}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:47:54,499] Trial 9 finished with value: 0.7603242728223003 and parameters: {'learning_rate': 0.047143207729416305, 'n_estimators': 108, 'max_depth': 9, 'min_child_weight': 63.19614509266372, 'gamma': 0.7465130945341614, 'subsample': 0.747154160908287, 'colsample_bytree': 0.6248637541610979, 'reg_alpha': 9.192411252082369, 'reg_lambda': 3.2111431919777877}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:48:32,878] Trial 10 finished with value: 0.7569976656005812 and parameters: {'learning_rate': 0.7891893833917394, 'n_estimators': 982, 'max_depth': 13, 'min_child_weight': 113.17826948433762, 'gamma': 0.4028121729781956, 'subsample': 0.5035002389481237, 'colsample_bytree': 0.9918953002210494, 'reg_alpha': 13.501342178388683, 'reg_lambda': 29.666662198877074}. Best is trial 4 with value: 0.7789660106529739.\n",
      "[I 2024-04-28 18:48:48,286] Trial 11 finished with value: 0.778999848695285 and parameters: {'learning_rate': 0.3642254101618262, 'n_estimators': 697, 'max_depth': 15, 'min_child_weight': 95.88165691508902, 'gamma': 0.9670476203439888, 'subsample': 0.8731940218257793, 'colsample_bytree': 0.8022436739788027, 'reg_alpha': 11.306341407782067, 'reg_lambda': 21.072794309832748}. Best is trial 11 with value: 0.778999848695285.\n",
      "[I 2024-04-28 18:49:01,994] Trial 12 finished with value: 0.7758165256433578 and parameters: {'learning_rate': 0.43200945514543465, 'n_estimators': 625, 'max_depth': 21, 'min_child_weight': 101.25885470362266, 'gamma': 0.9930685768426454, 'subsample': 0.8607283964300452, 'colsample_bytree': 0.7854362272034612, 'reg_alpha': 11.812443452738655, 'reg_lambda': 19.908921195565103}. Best is trial 11 with value: 0.778999848695285.\n",
      "[I 2024-04-28 18:49:35,714] Trial 13 finished with value: 0.7749697686013458 and parameters: {'learning_rate': 0.31253061619873834, 'n_estimators': 938, 'max_depth': 15, 'min_child_weight': 130.56353812229244, 'gamma': 0.3329597804127462, 'subsample': 0.603565385821835, 'colsample_bytree': 0.5337062583714561, 'reg_alpha': 19.57630373008537, 'reg_lambda': 21.03197624107266}. Best is trial 11 with value: 0.778999848695285.\n",
      "[I 2024-04-28 18:50:02,368] Trial 14 finished with value: 0.7801701012289811 and parameters: {'learning_rate': 0.05307934071583972, 'n_estimators': 622, 'max_depth': 8, 'min_child_weight': 4.146672116961014, 'gamma': 0.9120153182921177, 'subsample': 0.8681985588281447, 'colsample_bytree': 0.7180795296783139, 'reg_alpha': 8.898347838830068, 'reg_lambda': 29.965385906010468}. Best is trial 14 with value: 0.7801701012289811.\n",
      "[I 2024-04-28 18:50:18,408] Trial 15 finished with value: 0.7736163233935934 and parameters: {'learning_rate': 0.8339959449203539, 'n_estimators': 854, 'max_depth': 8, 'min_child_weight': 80.75922502955785, 'gamma': 0.9871295546386731, 'subsample': 0.8806709704270637, 'colsample_bytree': 0.7174240886159442, 'reg_alpha': 15.862560534318302, 'reg_lambda': 17.743891391349223}. Best is trial 14 with value: 0.7801701012289811.\n",
      "[I 2024-04-28 18:50:32,696] Trial 16 finished with value: 0.7755535538742067 and parameters: {'learning_rate': 0.2935398150191074, 'n_estimators': 651, 'max_depth': 3, 'min_child_weight': 1.7448377351600044, 'gamma': 0.8466994165391966, 'subsample': 0.8795125082900235, 'colsample_bytree': 0.5695167704571212, 'reg_alpha': 24.681365115989465, 'reg_lambda': 25.062551221231658}. Best is trial 14 with value: 0.7801701012289811.\n",
      "[I 2024-04-28 18:51:08,287] Trial 17 finished with value: 0.7782037616172268 and parameters: {'learning_rate': 0.028775287020735972, 'n_estimators': 808, 'max_depth': 10, 'min_child_weight': 129.71183547714548, 'gamma': 0.8707851580692237, 'subsample': 0.8234037031637872, 'colsample_bytree': 0.4282691381315782, 'reg_alpha': 7.870693871812585, 'reg_lambda': 17.430877486828834}. Best is trial 14 with value: 0.7801701012289811.\n",
      "[I 2024-04-28 18:51:27,481] Trial 18 finished with value: 0.7546279245253164 and parameters: {'learning_rate': 0.010515833932944875, 'n_estimators': 336, 'max_depth': 29, 'min_child_weight': 2.7891644026073834, 'gamma': 0.5817403237221639, 'subsample': 0.8999042899347526, 'colsample_bytree': 0.7289892500830585, 'reg_alpha': 15.745877621563903, 'reg_lambda': 25.975614003285607}. Best is trial 14 with value: 0.7801701012289811.\n",
      "[I 2024-04-28 18:51:41,109] Trial 19 finished with value: 0.7781145628866382 and parameters: {'learning_rate': 0.06724772300260844, 'n_estimators': 562, 'max_depth': 6, 'min_child_weight': 81.2532225480905, 'gamma': 0.9136337884072534, 'subsample': 0.9210092692478888, 'colsample_bytree': 0.8787325923379943, 'reg_alpha': 12.346750589160346, 'reg_lambda': 23.658665736741963}. Best is trial 14 with value: 0.7801701012289811.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7801701012289811\n",
      "Best Params: \n",
      "  learning_rate: 0.05307934071583972\n",
      "  n_estimators: 622\n",
      "  max_depth: 8\n",
      "  min_child_weight: 4.146672116961014\n",
      "  gamma: 0.9120153182921177\n",
      "  subsample: 0.8681985588281447\n",
      "  colsample_bytree: 0.7180795296783139\n",
      "  reg_alpha: 8.898347838830068\n",
      "  reg_lambda: 29.965385906010468\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_xgb(trial, X_train_distance_band10000, y_train_split, X_val_distance_band10000, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_xgb_dist10000.pkl')\n",
    "\n",
    "# print the best scores for each trial \n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77d99dce-f14d-462e-9b54-329585923014",
   "metadata": {},
   "source": [
    "# model:Tabnet_distanceband10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e2e36cbc-78c7-4cf2-a5b5-65b1a1e2e44f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 12:19:43,630] A new study created in memory with name: no-name-5ed58669-9a7a-4978-b1b1-5f020e1dff68\n",
      "[I 2024-04-29 12:21:57,870] Trial 0 finished with value: 0.6691699769253033 and parameters: {'mask_type': 'sparsemax', 'n_steps': 12, 'n_d': 4, 'gamma': 1.4, 'n_shared': 1, 'lambda_sparse': 0.00034461806382624863, 'patienceScheduler': 4}. Best is trial 0 with value: 0.6691699769253033.\n",
      "[I 2024-04-29 12:23:45,414] Trial 1 finished with value: 0.6800600655635922 and parameters: {'mask_type': 'sparsemax', 'n_steps': 7, 'n_d': 12, 'gamma': 1.6, 'n_shared': 1, 'lambda_sparse': 0.0004453385411137835, 'patienceScheduler': 9}. Best is trial 1 with value: 0.6800600655635922.\n",
      "[I 2024-04-29 12:26:29,883] Trial 2 finished with value: 0.7541825005118925 and parameters: {'mask_type': 'sparsemax', 'n_steps': 13, 'n_d': 6, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 5.0914270343159304e-05, 'patienceScheduler': 8}. Best is trial 2 with value: 0.7541825005118925.\n",
      "[I 2024-04-29 12:28:36,727] Trial 3 finished with value: 0.7584777580883267 and parameters: {'mask_type': 'entmax', 'n_steps': 9, 'n_d': 4, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 6.766351948340933e-05, 'patienceScheduler': 6}. Best is trial 3 with value: 0.7584777580883267.\n",
      "[I 2024-04-29 12:31:19,019] Trial 4 finished with value: 0.7573698295751344 and parameters: {'mask_type': 'sparsemax', 'n_steps': 11, 'n_d': 12, 'gamma': 1.0, 'n_shared': 4, 'lambda_sparse': 0.00048051618876580844, 'patienceScheduler': 6}. Best is trial 3 with value: 0.7584777580883267.\n",
      "[I 2024-04-29 12:33:06,131] Trial 5 finished with value: 0.7660396609888023 and parameters: {'mask_type': 'entmax', 'n_steps': 5, 'n_d': 8, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 0.0007131203334746554, 'patienceScheduler': 9}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:34:47,309] Trial 6 finished with value: 0.7535597950842823 and parameters: {'mask_type': 'entmax', 'n_steps': 4, 'n_d': 10, 'gamma': 1.6, 'n_shared': 4, 'lambda_sparse': 6.460963141194904e-05, 'patienceScheduler': 9}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:36:29,156] Trial 7 finished with value: 0.7538044762756718 and parameters: {'mask_type': 'entmax', 'n_steps': 7, 'n_d': 12, 'gamma': 1.4, 'n_shared': 1, 'lambda_sparse': 2.65400166388524e-05, 'patienceScheduler': 8}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:38:51,405] Trial 8 finished with value: 0.543781416907153 and parameters: {'mask_type': 'sparsemax', 'n_steps': 15, 'n_d': 10, 'gamma': 1.2, 'n_shared': 1, 'lambda_sparse': 1.1378875591506655e-05, 'patienceScheduler': 7}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:40:15,970] Trial 9 finished with value: 0.7490686460435444 and parameters: {'mask_type': 'sparsemax', 'n_steps': 3, 'n_d': 4, 'gamma': 1.2, 'n_shared': 1, 'lambda_sparse': 4.0132550583663625e-06, 'patienceScheduler': 4}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:42:05,319] Trial 10 finished with value: 0.7552542908371774 and parameters: {'mask_type': 'entmax', 'n_steps': 6, 'n_d': 8, 'gamma': 1.2, 'n_shared': 3, 'lambda_sparse': 1.00249382691051e-06, 'patienceScheduler': 10}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:44:09,821] Trial 11 finished with value: 0.6281533526319538 and parameters: {'mask_type': 'entmax', 'n_steps': 9, 'n_d': 2, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 0.00014071146258124484, 'patienceScheduler': 6}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:46:13,272] Trial 12 finished with value: 0.7622993251595631 and parameters: {'mask_type': 'entmax', 'n_steps': 9, 'n_d': 6, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 0.00085951397143325, 'patienceScheduler': 5}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:47:56,190] Trial 13 finished with value: 0.7653683859967804 and parameters: {'mask_type': 'entmax', 'n_steps': 5, 'n_d': 8, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 0.000943058278706644, 'patienceScheduler': 3}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:49:39,894] Trial 14 finished with value: 0.7637679894245949 and parameters: {'mask_type': 'entmax', 'n_steps': 5, 'n_d': 8, 'gamma': 1.2, 'n_shared': 3, 'lambda_sparse': 0.00020599273284288352, 'patienceScheduler': 3}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:51:11,041] Trial 15 finished with value: 0.760104068936619 and parameters: {'mask_type': 'entmax', 'n_steps': 3, 'n_d': 8, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 0.0007177409792213445, 'patienceScheduler': 3}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:52:59,452] Trial 16 finished with value: 0.7463401806666129 and parameters: {'mask_type': 'entmax', 'n_steps': 5, 'n_d': 10, 'gamma': 1.4, 'n_shared': 4, 'lambda_sparse': 0.000166786116687268, 'patienceScheduler': 7}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:54:57,048] Trial 17 finished with value: 0.7544560805326478 and parameters: {'mask_type': 'entmax', 'n_steps': 7, 'n_d': 6, 'gamma': 1.2, 'n_shared': 3, 'lambda_sparse': 1.825027084993605e-05, 'patienceScheduler': 10}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:56:36,915] Trial 18 finished with value: 0.7623053222412411 and parameters: {'mask_type': 'entmax', 'n_steps': 5, 'n_d': 8, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 0.0002466735606092675, 'patienceScheduler': 5}. Best is trial 5 with value: 0.7660396609888023.\n",
      "[I 2024-04-29 12:58:48,062] Trial 19 finished with value: 0.7560963099072033 and parameters: {'mask_type': 'entmax', 'n_steps': 8, 'n_d': 10, 'gamma': 1.2, 'n_shared': 4, 'lambda_sparse': 0.0009712957813324931, 'patienceScheduler': 8}. Best is trial 5 with value: 0.7660396609888023.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7660396609888023\n",
      "Best Params: \n",
      "  mask_type: entmax\n",
      "  n_steps: 5\n",
      "  n_d: 8\n",
      "  gamma: 1.0\n",
      "  n_shared: 3\n",
      "  lambda_sparse: 0.0007131203334746554\n",
      "  patienceScheduler: 9\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_tabnet(trial, X_train_distance_band10000, y_train_split, X_val_distance_band10000, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_tabnet_dist10000.pkl')\n",
    "\n",
    "# Retrieve the best trial and print its score and parameters\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "909061af-8a83-4ee5-a5fc-f859b901ca34",
   "metadata": {},
   "source": [
    "<a id='section6'></a>\n",
    "# ROOK\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b6d2b75-61fc-4888-b259-9d17175d5e51",
   "metadata": {},
   "source": [
    "# model:KNN_Rook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2fe871c0-3b73-4fb9-bed8-7f8efc04d3e0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 09:37:45,535] A new study created in memory with name: no-name-73298a2f-ef06-4677-9b58-1e904b0cf43f\n",
      "[I 2024-04-29 09:37:51,307] Trial 0 finished with value: 0.5320842590105673 and parameters: {'n_neighbors': 274, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'brute'}. Best is trial 0 with value: 0.5320842590105673.\n",
      "[I 2024-04-29 09:38:07,947] Trial 1 finished with value: 0.5848010985529151 and parameters: {'n_neighbors': 250, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:38:12,829] Trial 2 finished with value: 0.5288608644696342 and parameters: {'n_neighbors': 304, 'weights': 'distance', 'metric': 'minkowski', 'algorithm': 'brute'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:42:56,037] Trial 3 finished with value: 0.5833056403395671 and parameters: {'n_neighbors': 283, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'ball_tree'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:45:33,791] Trial 4 finished with value: 0.5285633380682357 and parameters: {'n_neighbors': 305, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'kd_tree'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:45:36,503] Trial 5 finished with value: 0.5293996204623594 and parameters: {'n_neighbors': 302, 'weights': 'distance', 'metric': 'euclidean', 'algorithm': 'brute'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:45:47,996] Trial 6 finished with value: 0.5824955157715229 and parameters: {'n_neighbors': 252, 'weights': 'uniform', 'metric': 'manhattan', 'algorithm': 'brute'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:45:58,979] Trial 7 finished with value: 0.5836427235675405 and parameters: {'n_neighbors': 274, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'brute'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:46:02,325] Trial 8 finished with value: 0.5242935104910253 and parameters: {'n_neighbors': 300, 'weights': 'uniform', 'metric': 'euclidean', 'algorithm': 'brute'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:50:23,439] Trial 9 finished with value: 0.5810702273312445 and parameters: {'n_neighbors': 275, 'weights': 'uniform', 'metric': 'manhattan', 'algorithm': 'kd_tree'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:50:26,406] Trial 10 finished with value: 0.5314590192852366 and parameters: {'n_neighbors': 250, 'weights': 'uniform', 'metric': 'minkowski', 'algorithm': 'auto'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:50:37,267] Trial 11 finished with value: 0.5846804100768581 and parameters: {'n_neighbors': 263, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 1 with value: 0.5848010985529151.\n",
      "[I 2024-04-29 09:50:49,305] Trial 12 finished with value: 0.5848029501467725 and parameters: {'n_neighbors': 261, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 12 with value: 0.5848029501467725.\n",
      "[I 2024-04-29 09:51:00,641] Trial 13 finished with value: 0.5846804100768581 and parameters: {'n_neighbors': 263, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 12 with value: 0.5848029501467725.\n",
      "[I 2024-04-29 09:51:12,847] Trial 14 finished with value: 0.5846677003678911 and parameters: {'n_neighbors': 260, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 12 with value: 0.5848029501467725.\n",
      "[I 2024-04-29 09:51:26,693] Trial 15 finished with value: 0.5848791583505746 and parameters: {'n_neighbors': 257, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 15 with value: 0.5848791583505746.\n",
      "[I 2024-04-29 09:55:07,946] Trial 16 finished with value: 0.5306724339566224 and parameters: {'n_neighbors': 289, 'weights': 'distance', 'metric': 'minkowski', 'algorithm': 'ball_tree'}. Best is trial 15 with value: 0.5848791583505746.\n",
      "[I 2024-04-29 09:55:18,609] Trial 17 finished with value: 0.5846677003678911 and parameters: {'n_neighbors': 260, 'weights': 'distance', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 15 with value: 0.5848791583505746.\n",
      "[I 2024-04-29 09:55:29,772] Trial 18 finished with value: 0.5810267476679 and parameters: {'n_neighbors': 268, 'weights': 'uniform', 'metric': 'manhattan', 'algorithm': 'auto'}. Best is trial 15 with value: 0.5848791583505746.\n",
      "[I 2024-04-29 09:55:32,472] Trial 19 finished with value: 0.5301686476131963 and parameters: {'n_neighbors': 293, 'weights': 'distance', 'metric': 'minkowski', 'algorithm': 'auto'}. Best is trial 15 with value: 0.5848791583505746.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.5848791583505746\n",
      "Best Params: \n",
      "  n_neighbors: 257\n",
      "  weights: distance\n",
      "  metric: manhattan\n",
      "  algorithm: auto\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "import joblib\n",
    "\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_knn(trial, X_train_rook, y_train_split, X_val_rook, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_knn_Rook.pkl')\n",
    "\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22caf680-5f79-499b-93ab-d56efcdd9cda",
   "metadata": {},
   "source": [
    "# model:Rf_Rook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "33ba696b-c2ad-48ff-8418-59fbd7f3e277",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 15:32:57,069] A new study created in memory with name: no-name-fb4243c9-40cf-456b-bee4-1a227f54ab63\n",
      "[I 2024-04-29 15:32:58,454] Trial 0 finished with value: 0.7227262670866385 and parameters: {'n_estimators': 7, 'max_depth': 31, 'min_samples_leaf': 5, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 0 with value: 0.7227262670866385.\n",
      "[I 2024-04-29 15:32:59,329] Trial 1 finished with value: 0.6860352960378734 and parameters: {'n_estimators': 7, 'max_depth': 8, 'min_samples_leaf': 7, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 0 with value: 0.7227262670866385.\n",
      "[I 2024-04-29 15:33:00,939] Trial 2 finished with value: 0.6449258713318307 and parameters: {'n_estimators': 6, 'max_depth': 4, 'min_samples_leaf': 7, 'max_features': None, 'bootstrap': True, 'criterion': 'gini'}. Best is trial 0 with value: 0.7227262670866385.\n",
      "[I 2024-04-29 15:33:56,914] Trial 3 finished with value: 0.7026152954003206 and parameters: {'n_estimators': 40, 'max_depth': 38, 'min_samples_leaf': 9, 'max_features': None, 'bootstrap': False, 'criterion': 'gini'}. Best is trial 0 with value: 0.7227262670866385.\n",
      "[I 2024-04-29 15:33:57,743] Trial 4 finished with value: 0.6979719857736874 and parameters: {'n_estimators': 5, 'max_depth': 40, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 0 with value: 0.7227262670866385.\n",
      "[I 2024-04-29 15:34:00,884] Trial 5 finished with value: 0.7313829113320519 and parameters: {'n_estimators': 3, 'max_depth': 15, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': True, 'criterion': 'gini'}. Best is trial 5 with value: 0.7313829113320519.\n",
      "[I 2024-04-29 15:34:01,118] Trial 6 finished with value: 0.6498572458310928 and parameters: {'n_estimators': 1, 'max_depth': 16, 'min_samples_leaf': 9, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 5 with value: 0.7313829113320519.\n",
      "[I 2024-04-29 15:34:05,275] Trial 7 finished with value: 0.7256291442734987 and parameters: {'n_estimators': 3, 'max_depth': 38, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': True, 'criterion': 'entropy'}. Best is trial 5 with value: 0.7313829113320519.\n",
      "[I 2024-04-29 15:34:17,970] Trial 8 finished with value: 0.7035331001576359 and parameters: {'n_estimators': 20, 'max_depth': 7, 'min_samples_leaf': 7, 'max_features': None, 'bootstrap': False, 'criterion': 'gini'}. Best is trial 5 with value: 0.7313829113320519.\n",
      "[I 2024-04-29 15:34:36,687] Trial 9 finished with value: 0.754666916621825 and parameters: {'n_estimators': 19, 'max_depth': 14, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': True, 'criterion': 'gini'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:35:34,663] Trial 10 finished with value: 0.7336696552499943 and parameters: {'n_estimators': 216, 'max_depth': 24, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:36:53,573] Trial 11 finished with value: 0.7349042176331759 and parameters: {'n_estimators': 296, 'max_depth': 25, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:38:10,419] Trial 12 finished with value: 0.7345680086335991 and parameters: {'n_estimators': 290, 'max_depth': 24, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:38:26,879] Trial 13 finished with value: 0.7323687328944037 and parameters: {'n_estimators': 66, 'max_depth': 18, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:38:40,664] Trial 14 finished with value: 0.7321177566101471 and parameters: {'n_estimators': 72, 'max_depth': 30, 'min_samples_leaf': 9, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:40:14,951] Trial 15 finished with value: 0.7172968539045576 and parameters: {'n_estimators': 26, 'max_depth': 11, 'min_samples_leaf': 5, 'max_features': None, 'bootstrap': False, 'criterion': 'entropy'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:42:02,549] Trial 16 finished with value: 0.7351476830798239 and parameters: {'n_estimators': 123, 'max_depth': 24, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': False, 'criterion': 'gini'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:42:48,531] Trial 17 finished with value: 0.7109728600398251 and parameters: {'n_estimators': 96, 'max_depth': 12, 'min_samples_leaf': 5, 'max_features': 'log2', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:55:11,300] Trial 18 finished with value: 0.6943376516730112 and parameters: {'n_estimators': 123, 'max_depth': 20, 'min_samples_leaf': 7, 'max_features': None, 'bootstrap': False, 'criterion': 'gini'}. Best is trial 9 with value: 0.754666916621825.\n",
      "[I 2024-04-29 15:55:12,521] Trial 19 finished with value: 0.3601527189180479 and parameters: {'n_estimators': 16, 'max_depth': 1, 'min_samples_leaf': 9, 'max_features': 'sqrt', 'bootstrap': True, 'criterion': 'gini'}. Best is trial 9 with value: 0.754666916621825.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.754666916621825\n",
      "Best Params: \n",
      "  n_estimators: 19\n",
      "  max_depth: 14\n",
      "  min_samples_leaf: 5\n",
      "  max_features: None\n",
      "  bootstrap: True\n",
      "  criterion: gini\n"
     ]
    }
   ],
   "source": [
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_rf(trial, X_train_rook, y_train_split, X_val_rook, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_rf_Rook.pkl')\n",
    "\n",
    "\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "160b5f58-1986-42de-af0a-227bcf9d93b1",
   "metadata": {},
   "source": [
    "# model:XGB_Rook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "99520b5f-5669-4043-ba7e-b5584bca390a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-28 18:51:41,141] A new study created in memory with name: no-name-0b082c84-37df-40d4-a762-6e51cb0adc75\n",
      "[I 2024-04-28 18:51:57,310] Trial 0 finished with value: 0.7534668464295473 and parameters: {'learning_rate': 0.047012420409594906, 'n_estimators': 867, 'max_depth': 3, 'min_child_weight': 171.26132202716326, 'gamma': 0.2953018640795896, 'subsample': 0.7616489416230428, 'colsample_bytree': 0.17849717722595418, 'reg_alpha': 17.938942164583807, 'reg_lambda': 11.0776380628053}. Best is trial 0 with value: 0.7534668464295473.\n",
      "[I 2024-04-28 18:52:02,106] Trial 1 finished with value: 0.7675901112458959 and parameters: {'learning_rate': 0.43851545028063416, 'n_estimators': 167, 'max_depth': 11, 'min_child_weight': 159.39504824223584, 'gamma': 0.33305522444674496, 'subsample': 0.6122001015115437, 'colsample_bytree': 0.3313944331827162, 'reg_alpha': 1.0311656524187318, 'reg_lambda': 12.12675814723936}. Best is trial 1 with value: 0.7675901112458959.\n",
      "[I 2024-04-28 18:52:13,398] Trial 2 finished with value: 0.769989858874561 and parameters: {'learning_rate': 0.033219458722018515, 'n_estimators': 554, 'max_depth': 4, 'min_child_weight': 4.046827705255192, 'gamma': 0.9939808555250214, 'subsample': 0.9177826417946295, 'colsample_bytree': 0.9425204384357917, 'reg_alpha': 10.421421280497002, 'reg_lambda': 27.146598297269342}. Best is trial 2 with value: 0.769989858874561.\n",
      "[I 2024-04-28 18:52:17,460] Trial 3 finished with value: 0.765024334138173 and parameters: {'learning_rate': 0.7615997946935755, 'n_estimators': 170, 'max_depth': 12, 'min_child_weight': 38.35332813571647, 'gamma': 0.9410592423128442, 'subsample': 0.8292929736644461, 'colsample_bytree': 0.8903884218038256, 'reg_alpha': 1.283681264432086, 'reg_lambda': 9.387846317606021}. Best is trial 2 with value: 0.769989858874561.\n",
      "[I 2024-04-28 18:52:34,319] Trial 4 finished with value: 0.7594453692886345 and parameters: {'learning_rate': 0.012917206831972445, 'n_estimators': 425, 'max_depth': 12, 'min_child_weight': 172.27867068115032, 'gamma': 0.09418041483005013, 'subsample': 0.6479823819253505, 'colsample_bytree': 0.7068225939869834, 'reg_alpha': 11.127297366196473, 'reg_lambda': 28.534889306962857}. Best is trial 2 with value: 0.769989858874561.\n",
      "[I 2024-04-28 18:52:47,351] Trial 5 finished with value: 0.7726472878314353 and parameters: {'learning_rate': 0.4023160962996854, 'n_estimators': 394, 'max_depth': 15, 'min_child_weight': 45.52872205167658, 'gamma': 0.6140222548540267, 'subsample': 0.5048851287771942, 'colsample_bytree': 0.5531589945378301, 'reg_alpha': 10.811823215286086, 'reg_lambda': 0.4418248685169557}. Best is trial 5 with value: 0.7726472878314353.\n",
      "[I 2024-04-28 18:53:01,199] Trial 6 finished with value: 0.7752488141128395 and parameters: {'learning_rate': 0.3202565291605288, 'n_estimators': 746, 'max_depth': 13, 'min_child_weight': 58.188622929897285, 'gamma': 0.6407701973270361, 'subsample': 0.9149891895297046, 'colsample_bytree': 0.7478098913424319, 'reg_alpha': 23.31788718511901, 'reg_lambda': 29.154804504750423}. Best is trial 6 with value: 0.7752488141128395.\n",
      "[I 2024-04-28 18:53:13,474] Trial 7 finished with value: 0.7780626351728168 and parameters: {'learning_rate': 0.18076395802571862, 'n_estimators': 483, 'max_depth': 29, 'min_child_weight': 66.09019540655768, 'gamma': 0.38440896267877755, 'subsample': 0.941170029716947, 'colsample_bytree': 0.4085014222080714, 'reg_alpha': 28.569631463302898, 'reg_lambda': 1.1376947982786323}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:53:20,682] Trial 8 finished with value: 0.7764027412383645 and parameters: {'learning_rate': 0.1043511976310042, 'n_estimators': 164, 'max_depth': 13, 'min_child_weight': 141.1077729180373, 'gamma': 0.12107625450047099, 'subsample': 0.6932978157790073, 'colsample_bytree': 0.9804126187301141, 'reg_alpha': 11.73052806911822, 'reg_lambda': 28.229985072070527}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:53:51,350] Trial 9 finished with value: 0.7744252249019833 and parameters: {'learning_rate': 0.02425615329339887, 'n_estimators': 860, 'max_depth': 8, 'min_child_weight': 46.47793212964443, 'gamma': 0.2570659152876692, 'subsample': 0.5491858375202467, 'colsample_bytree': 0.8297057420141436, 'reg_alpha': 29.97747941486429, 'reg_lambda': 29.8759931240182}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:54:05,973] Trial 10 finished with value: 0.7764360044457932 and parameters: {'learning_rate': 0.12326728642140636, 'n_estimators': 605, 'max_depth': 29, 'min_child_weight': 101.3439346637389, 'gamma': 0.5013180375524847, 'subsample': 0.9977998027536737, 'colsample_bytree': 0.41066707723103424, 'reg_alpha': 29.738369576496144, 'reg_lambda': 0.13971330062153342}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:54:20,043] Trial 11 finished with value: 0.7770503374110861 and parameters: {'learning_rate': 0.13842024382957954, 'n_estimators': 604, 'max_depth': 30, 'min_child_weight': 99.94205075315071, 'gamma': 0.4901578617772488, 'subsample': 0.9959721064503418, 'colsample_bytree': 0.40838557337545667, 'reg_alpha': 29.979748227931445, 'reg_lambda': 0.4775410914624134}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:54:29,350] Trial 12 finished with value: 0.7754319117418292 and parameters: {'learning_rate': 0.18327327453439177, 'n_estimators': 417, 'max_depth': 30, 'min_child_weight': 101.14570118635035, 'gamma': 0.7104840353896249, 'subsample': 0.9893028364796884, 'colsample_bytree': 0.4919927124632556, 'reg_alpha': 23.887871296393065, 'reg_lambda': 4.825383724210267}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:54:52,101] Trial 13 finished with value: 0.7721922737864595 and parameters: {'learning_rate': 0.06219548221881002, 'n_estimators': 696, 'max_depth': 24, 'min_child_weight': 80.10888363432488, 'gamma': 0.40405828897899165, 'subsample': 0.8434379761874498, 'colsample_bytree': 0.22826813823595282, 'reg_alpha': 24.547902536650437, 'reg_lambda': 18.583775460579155}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:55:00,874] Trial 14 finished with value: 0.776407569966928 and parameters: {'learning_rate': 0.19966138904631076, 'n_estimators': 319, 'max_depth': 22, 'min_child_weight': 119.49130799222218, 'gamma': 0.5107974944232985, 'subsample': 0.9015556266660902, 'colsample_bytree': 0.5907421424107385, 'reg_alpha': 19.139879066348335, 'reg_lambda': 5.053629280946769}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:55:20,989] Trial 15 finished with value: 0.7758356381046052 and parameters: {'learning_rate': 0.0664350746243712, 'n_estimators': 676, 'max_depth': 25, 'min_child_weight': 76.89584311720012, 'gamma': 0.7958309114207378, 'subsample': 0.7996542262298377, 'colsample_bytree': 0.30472714694463215, 'reg_alpha': 26.549721866789778, 'reg_lambda': 17.6018057334418}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:55:33,344] Trial 16 finished with value: 0.7753500530499504 and parameters: {'learning_rate': 0.18567187496328927, 'n_estimators': 517, 'max_depth': 19, 'min_child_weight': 8.512906247934211, 'gamma': 0.42088552936023677, 'subsample': 0.9486744061773441, 'colsample_bytree': 0.41283293884838024, 'reg_alpha': 19.388727521461067, 'reg_lambda': 4.881543645289105}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:55:50,954] Trial 17 finished with value: 0.7708658180155338 and parameters: {'learning_rate': 0.5829707996915715, 'n_estimators': 977, 'max_depth': 26, 'min_child_weight': 130.19444199522312, 'gamma': 0.19676865097320007, 'subsample': 0.8606173661329315, 'colsample_bytree': 0.10131343375824614, 'reg_alpha': 27.1521497884602, 'reg_lambda': 7.566572331474417}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:56:03,230] Trial 18 finished with value: 0.7747738794103777 and parameters: {'learning_rate': 0.24004038084813423, 'n_estimators': 312, 'max_depth': 20, 'min_child_weight': 196.48717587470452, 'gamma': 0.025359210169904944, 'subsample': 0.9603912076560175, 'colsample_bytree': 0.6531867422427936, 'reg_alpha': 6.6685123899879635, 'reg_lambda': 1.9329259699355248}. Best is trial 7 with value: 0.7780626351728168.\n",
      "[I 2024-04-28 18:56:18,250] Trial 19 finished with value: 0.7758209666724041 and parameters: {'learning_rate': 0.12135541660493297, 'n_estimators': 549, 'max_depth': 28, 'min_child_weight': 83.4358594330198, 'gamma': 0.5494636248016901, 'subsample': 0.7401899381259449, 'colsample_bytree': 0.4972973005599483, 'reg_alpha': 21.40429086701912, 'reg_lambda': 22.65448273606242}. Best is trial 7 with value: 0.7780626351728168.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.7780626351728168\n",
      "Best Params: \n",
      "  learning_rate: 0.18076395802571862\n",
      "  n_estimators: 483\n",
      "  max_depth: 29\n",
      "  min_child_weight: 66.09019540655768\n",
      "  gamma: 0.38440896267877755\n",
      "  subsample: 0.941170029716947\n",
      "  colsample_bytree: 0.4085014222080714\n",
      "  reg_alpha: 28.569631463302898\n",
      "  reg_lambda: 1.1376947982786323\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_xgb(trial, X_train_rook, y_train_split, X_val_rook, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_xgb_Rook.pkl')\n",
    "\n",
    "# print the best scores for each trial \n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6539768-9388-48f6-bd72-f9f30770101c",
   "metadata": {},
   "source": [
    "# model:Tabnet_Rook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3a717f1b-b2ad-4d73-9d98-7074afbf578e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-04-29 12:58:48,078] A new study created in memory with name: no-name-426a3135-8850-48ba-ad9f-ab0f9c14ce4d\n",
      "[I 2024-04-29 13:01:17,713] Trial 0 finished with value: 0.619570730193577 and parameters: {'mask_type': 'entmax', 'n_steps': 13, 'n_d': 4, 'gamma': 1.6, 'n_shared': 2, 'lambda_sparse': 2.0386951718262475e-05, 'patienceScheduler': 3}. Best is trial 0 with value: 0.619570730193577.\n",
      "[I 2024-04-29 13:03:14,469] Trial 1 finished with value: 0.674254365043565 and parameters: {'mask_type': 'sparsemax', 'n_steps': 7, 'n_d': 12, 'gamma': 1.4, 'n_shared': 3, 'lambda_sparse': 3.7451137267512475e-06, 'patienceScheduler': 8}. Best is trial 1 with value: 0.674254365043565.\n",
      "[I 2024-04-29 13:07:12,553] Trial 2 finished with value: 0.768637306153877 and parameters: {'mask_type': 'entmax', 'n_steps': 12, 'n_d': 6, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 0.0008914871643038309, 'patienceScheduler': 9}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:12:26,978] Trial 3 finished with value: 0.6874136585726346 and parameters: {'mask_type': 'sparsemax', 'n_steps': 5, 'n_d': 8, 'gamma': 1.2, 'n_shared': 1, 'lambda_sparse': 0.00010888757356298915, 'patienceScheduler': 7}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:18:02,176] Trial 4 finished with value: 0.7494765735385589 and parameters: {'mask_type': 'entmax', 'n_steps': 5, 'n_d': 12, 'gamma': 1.6, 'n_shared': 2, 'lambda_sparse': 8.015242538742864e-05, 'patienceScheduler': 8}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:25:17,618] Trial 5 finished with value: 0.7511311133951869 and parameters: {'mask_type': 'entmax', 'n_steps': 8, 'n_d': 10, 'gamma': 1.4, 'n_shared': 3, 'lambda_sparse': 1.0280985554020631e-05, 'patienceScheduler': 6}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:32:17,837] Trial 6 finished with value: 0.7510999952809212 and parameters: {'mask_type': 'sparsemax', 'n_steps': 11, 'n_d': 4, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 0.0003042098624639302, 'patienceScheduler': 10}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:39:45,233] Trial 7 finished with value: 0.5651841462233518 and parameters: {'mask_type': 'sparsemax', 'n_steps': 12, 'n_d': 4, 'gamma': 1.4, 'n_shared': 1, 'lambda_sparse': 1.907422057775444e-05, 'patienceScheduler': 5}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:46:43,630] Trial 8 finished with value: 0.6478869305491604 and parameters: {'mask_type': 'sparsemax', 'n_steps': 11, 'n_d': 6, 'gamma': 1.4, 'n_shared': 1, 'lambda_sparse': 3.917535996988078e-05, 'patienceScheduler': 3}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 13:54:51,531] Trial 9 finished with value: 0.6039120759319976 and parameters: {'mask_type': 'sparsemax', 'n_steps': 12, 'n_d': 4, 'gamma': 1.6, 'n_shared': 2, 'lambda_sparse': 9.362397425515994e-05, 'patienceScheduler': 4}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:06:24,907] Trial 10 finished with value: 0.7449711807363435 and parameters: {'mask_type': 'entmax', 'n_steps': 15, 'n_d': 8, 'gamma': 1.0, 'n_shared': 4, 'lambda_sparse': 0.0008858239881226193, 'patienceScheduler': 10}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:08:38,514] Trial 11 finished with value: 0.7494388064236531 and parameters: {'mask_type': 'entmax', 'n_steps': 8, 'n_d': 10, 'gamma': 1.2, 'n_shared': 3, 'lambda_sparse': 3.333369733076336e-06, 'patienceScheduler': 6}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:11:06,545] Trial 12 finished with value: 0.7363510486902001 and parameters: {'mask_type': 'entmax', 'n_steps': 9, 'n_d': 10, 'gamma': 1.2, 'n_shared': 4, 'lambda_sparse': 1.0432706008761674e-06, 'patienceScheduler': 8}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:12:41,720] Trial 13 finished with value: 0.7596691805859619 and parameters: {'mask_type': 'entmax', 'n_steps': 3, 'n_d': 6, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 8.192429573541163e-06, 'patienceScheduler': 6}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:14:15,827] Trial 14 finished with value: 0.6205008108793931 and parameters: {'mask_type': 'entmax', 'n_steps': 3, 'n_d': 2, 'gamma': 1.0, 'n_shared': 3, 'lambda_sparse': 0.0004158101215206644, 'patienceScheduler': 9}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:17:04,202] Trial 15 finished with value: 0.7633137284337673 and parameters: {'mask_type': 'entmax', 'n_steps': 15, 'n_d': 6, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 5.995553959312516e-06, 'patienceScheduler': 5}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:19:52,959] Trial 16 finished with value: 0.6830892643296196 and parameters: {'mask_type': 'entmax', 'n_steps': 15, 'n_d': 6, 'gamma': 1.2, 'n_shared': 2, 'lambda_sparse': 3.0888318190201555e-06, 'patienceScheduler': 5}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:22:18,173] Trial 17 finished with value: 0.6365717750875345 and parameters: {'mask_type': 'entmax', 'n_steps': 14, 'n_d': 2, 'gamma': 1.0, 'n_shared': 1, 'lambda_sparse': 0.00020489214792738188, 'patienceScheduler': 7}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:24:32,806] Trial 18 finished with value: 0.7540381012680768 and parameters: {'mask_type': 'entmax', 'n_steps': 10, 'n_d': 8, 'gamma': 1.0, 'n_shared': 2, 'lambda_sparse': 0.0009743435670933506, 'patienceScheduler': 5}. Best is trial 2 with value: 0.768637306153877.\n",
      "[I 2024-04-29 14:26:53,915] Trial 19 finished with value: 0.6677542315790373 and parameters: {'mask_type': 'entmax', 'n_steps': 13, 'n_d': 6, 'gamma': 1.2, 'n_shared': 1, 'lambda_sparse': 1.7307091032124955e-06, 'patienceScheduler': 9}. Best is trial 2 with value: 0.768637306153877.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Score:  0.768637306153877\n",
      "Best Params: \n",
      "  mask_type: entmax\n",
      "  n_steps: 12\n",
      "  n_d: 6\n",
      "  gamma: 1.0\n",
      "  n_shared: 1\n",
      "  lambda_sparse: 0.0008914871643038309\n",
      "  patienceScheduler: 9\n"
     ]
    }
   ],
   "source": [
    "import optuna\n",
    "random_state = 0\n",
    "\n",
    "pruner = optuna.pruners.SuccessiveHalvingPruner(min_resource=1, reduction_factor=3)\n",
    "study = optuna.create_study(direction='maximize', pruner=pruner)\n",
    "study.optimize(lambda trial: Objective_tabnet(trial, X_train_rook, y_train_split, X_val_rook, y_val), n_trials=20)\n",
    "\n",
    "# Save the study for later use\n",
    "joblib.dump(study, 'study_tabnet_ROOK.pkl')\n",
    "\n",
    "# Retrieve the best trial and print its score and parameters\n",
    "best_trial = study.best_trial\n",
    "print(\"Best Score: \", best_trial.value)\n",
    "print(\"Best Params: \")\n",
    "for key, value in best_trial.params.items():\n",
    "    print(\"  {}: {}\".format(key, value))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9cf6fa6-16e2-41b7-8cb6-4dec3f4cf565",
   "metadata": {},
   "source": [
    "<a id='section7'></a>\n",
    "# Predictions\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a3303d17-b166-46ac-855e-818d9a9864f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the parameters of baseline knn and get the predictions \n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_parm_10000 =joblib.load('study_knn_dist10000.pkl')\n",
    "\n",
    "y_pred_knn_10000, y_pred_prob_knn_10000 = predict(classifier=KNeighborsClassifier,\n",
    "                          params=knn_parm_10000.best_params, X_train=X_train_distance_band10000, y_train=y_train_split,\n",
    "                                                  X_test=X_test_distance_band10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d06ec58c-5c98-46d2-99a7-febb32210f6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the parameters of baseline knn and get the predictions \n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_parm_KNN8 =joblib.load('study_knn_KNN8.pkl')\n",
    "\n",
    "y_pred_knn_KNN8, y_pred_prob_knn_KNN8 = predict(classifier=KNeighborsClassifier,\n",
    "                          params=knn_parm_KNN8.best_params, X_train=X_train_KNN8 , y_train=y_train_split, \n",
    "                          X_test=X_test_KNN8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "748b914d-72fc-4737-a9f8-b51d2d6c5659",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "import joblib\n",
    "knn_parm_rook = joblib.load('study_knn_Rook.pkl')\n",
    "\n",
    "y_pred_KNN_rook, y_pred_prob_KNN_rook = predict(classifier=KNeighborsClassifier,\n",
    "                                                                  params=knn_parm_rook.best_params,\n",
    "                                                                  X_train=X_train_rook,\n",
    "                                                                  y_train=y_train_split, \n",
    "                                                                  X_test=X_test_rook)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "72909615-decb-4533-bdcb-d26103fe1a9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import joblib\n",
    "\n",
    "rf_parm_10000 = joblib.load('study_rf_dist10000.pkl')\n",
    "y_pred_rf_10000, y_pred_prob_rf_10000 = predict(classifier=RandomForestClassifier,\n",
    "                                                                params=rf_parm_10000.best_params,  \n",
    "                                                                X_train=X_train_distance_band10000, \n",
    "                                                                y_train=y_train_split, \n",
    "                                                                X_test=X_test_distance_band10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ebcac0a8-fce9-454e-9fc0-8f1173dc4007",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_parm_KNN8 = joblib.load('study_rf_KNN8.pkl')\n",
    "\n",
    "\n",
    "y_pred_rf_KNN8, y_pred_prob_rf_KNN8 = predict(classifier=RandomForestClassifier,\n",
    "                                  params=rf_parm_KNN8.best_params,\n",
    "                                  X_train=X_train_KNN8,\n",
    "                                  y_train=y_train_split,\n",
    "                                  X_test=X_test_KNN8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "ceacca92-7924-4b9e-b326-8740b0fed871",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_parm_rook = joblib.load('study_rf_Rook.pkl')\n",
    "y_pred_rf_rook, y_pred_prob_rf_rook = predict(classifier=RandomForestClassifier,\n",
    "                                   params=rf_parm_rook.best_params,\n",
    "                                   X_train=X_train_rook,\n",
    "                                   y_train=y_train_split,\n",
    "                                   X_test=X_test_rook)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7bdea884-34c1-4988-8fcd-b056773376eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "xgb_model_rook = joblib.load('study_xgb_Rook.pkl')\n",
    "y_pred_xgb_rook, y_pred_prob_xgb_rook = predict(classifier=xgb.XGBClassifier,\n",
    "                                                params=xgb_model_rook.best_params,\n",
    "                                                X_train=X_train_rook,\n",
    "                                                y_train=y_train_split,\n",
    "                                                X_test=X_test_rook)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "5a4beeb5-830a-40dc-9c3c-8b1ba5ceb678",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "xgb_model_KNN8 = joblib.load('study_xgb_KNN8.pkl')\n",
    "\n",
    "y_pred_xgb_KNN8, y_pred_prob_xgb_KNN8 = predict(classifier=xgb.XGBClassifier, \n",
    "                                                params=xgb_model_KNN8.best_params,\n",
    "                                                X_train=X_train_KNN8,\n",
    "                                                y_train=y_train_split,\n",
    "                                                X_test=X_test_KNN8)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2653a87e-64e9-4bee-9c48-fc7f866876da",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "xgb_model_10000 = joblib.load('study_xgb_dist10000.pkl')\n",
    "y_pred_xgb_10000, y_pred_prob_xgb_10000 = predict(classifier=xgb.XGBClassifier,\n",
    "                                                  params=xgb_model_10000.best_params,\n",
    "                                                  X_train=X_train_distance_band10000,\n",
    "                                                  y_train=y_train_split,\n",
    "                                                  X_test=X_test_distance_band10000)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c841d81a-fe93-48f0-8b37-63ce503d5866",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mask_type': 'entmax',\n",
       " 'n_steps': 5,\n",
       " 'n_d': 8,\n",
       " 'gamma': 1.0,\n",
       " 'n_shared': 3,\n",
       " 'lambda_sparse': 0.0007131203334746554,\n",
       " 'patienceScheduler': 9}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tabnet_model_10000 = joblib.load('study_tabnet_dist10000.pkl')\n",
    "tabnet_model_10000.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6117c69e-4fd0-4ae5-8289-914aa1bb3b5b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  | loss: 1.1014  |  0:00:07s\n",
      "epoch 1  | loss: 0.68863 |  0:00:15s\n",
      "epoch 2  | loss: 0.62139 |  0:00:22s\n",
      "epoch 3  | loss: 0.59648 |  0:00:30s\n",
      "epoch 4  | loss: 0.58219 |  0:00:37s\n",
      "epoch 5  | loss: 0.57902 |  0:00:45s\n",
      "epoch 6  | loss: 0.56817 |  0:00:52s\n",
      "epoch 7  | loss: 0.56294 |  0:01:00s\n",
      "epoch 8  | loss: 0.55668 |  0:01:07s\n",
      "epoch 9  | loss: 0.55037 |  0:01:14s\n",
      "epoch 10 | loss: 0.54806 |  0:01:22s\n",
      "epoch 11 | loss: 0.5439  |  0:01:29s\n",
      "epoch 12 | loss: 0.54518 |  0:01:37s\n",
      "epoch 13 | loss: 0.54001 |  0:01:44s\n",
      "epoch 14 | loss: 0.5382  |  0:01:51s\n",
      "epoch 15 | loss: 0.53956 |  0:01:59s\n",
      "epoch 16 | loss: 0.53305 |  0:02:06s\n",
      "epoch 17 | loss: 0.53688 |  0:02:14s\n",
      "epoch 18 | loss: 0.5364  |  0:02:22s\n",
      "epoch 19 | loss: 0.53129 |  0:02:30s\n",
      "epoch 20 | loss: 0.53145 |  0:02:37s\n",
      "epoch 21 | loss: 0.53168 |  0:02:45s\n",
      "epoch 22 | loss: 0.52815 |  0:02:52s\n",
      "epoch 23 | loss: 0.52411 |  0:03:00s\n",
      "epoch 24 | loss: 0.52277 |  0:03:08s\n",
      "epoch 25 | loss: 0.52102 |  0:03:15s\n",
      "epoch 26 | loss: 0.51942 |  0:03:23s\n",
      "epoch 27 | loss: 0.52056 |  0:03:31s\n",
      "epoch 28 | loss: 0.51938 |  0:03:39s\n",
      "epoch 29 | loss: 0.52243 |  0:03:47s\n",
      "epoch 30 | loss: 0.52031 |  0:03:55s\n",
      "epoch 31 | loss: 0.51856 |  0:04:02s\n",
      "epoch 32 | loss: 0.51465 |  0:04:11s\n",
      "epoch 33 | loss: 0.51431 |  0:04:19s\n",
      "epoch 34 | loss: 0.51324 |  0:04:26s\n",
      "epoch 35 | loss: 0.5115  |  0:04:35s\n",
      "epoch 36 | loss: 0.51471 |  0:04:43s\n",
      "epoch 37 | loss: 0.51518 |  0:04:50s\n",
      "epoch 38 | loss: 0.51498 |  0:04:58s\n",
      "epoch 39 | loss: 0.52011 |  0:05:06s\n",
      "epoch 40 | loss: 0.51305 |  0:05:14s\n",
      "epoch 41 | loss: 0.50757 |  0:05:21s\n",
      "epoch 42 | loss: 0.507   |  0:05:29s\n",
      "epoch 43 | loss: 0.50203 |  0:05:37s\n",
      "epoch 44 | loss: 0.50565 |  0:05:45s\n",
      "epoch 45 | loss: 0.50416 |  0:05:52s\n",
      "epoch 46 | loss: 0.50152 |  0:06:00s\n",
      "epoch 47 | loss: 0.50194 |  0:06:08s\n",
      "epoch 48 | loss: 0.50036 |  0:06:15s\n",
      "epoch 49 | loss: 0.50749 |  0:06:23s\n",
      "epoch 50 | loss: 0.50527 |  0:06:31s\n",
      "epoch 51 | loss: 0.50318 |  0:06:38s\n",
      "epoch 52 | loss: 0.49827 |  0:06:46s\n",
      "epoch 53 | loss: 0.49616 |  0:06:54s\n",
      "epoch 54 | loss: 0.49986 |  0:07:02s\n",
      "epoch 55 | loss: 0.49047 |  0:07:10s\n",
      "epoch 56 | loss: 0.50935 |  0:07:17s\n",
      "epoch 57 | loss: 0.50704 |  0:07:25s\n",
      "epoch 58 | loss: 0.50469 |  0:07:33s\n",
      "epoch 59 | loss: 0.51263 |  0:07:40s\n",
      "epoch 60 | loss: 0.50367 |  0:07:48s\n",
      "epoch 61 | loss: 0.50529 |  0:07:56s\n",
      "epoch 62 | loss: 0.50864 |  0:08:04s\n",
      "epoch 63 | loss: 0.50513 |  0:08:12s\n",
      "epoch 64 | loss: 0.49973 |  0:08:20s\n",
      "epoch 65 | loss: 0.49891 |  0:08:29s\n",
      "epoch 66 | loss: 0.49844 |  0:08:37s\n",
      "epoch 67 | loss: 0.50249 |  0:08:44s\n",
      "epoch 68 | loss: 0.5009  |  0:08:52s\n",
      "epoch 69 | loss: 0.49422 |  0:09:00s\n",
      "epoch 70 | loss: 0.4948  |  0:09:08s\n",
      "epoch 71 | loss: 0.48927 |  0:09:16s\n",
      "epoch 72 | loss: 0.48629 |  0:09:24s\n",
      "epoch 73 | loss: 0.48542 |  0:09:31s\n",
      "epoch 74 | loss: 0.4846  |  0:09:40s\n",
      "epoch 75 | loss: 0.48993 |  0:09:47s\n",
      "epoch 76 | loss: 0.48996 |  0:09:56s\n",
      "epoch 77 | loss: 0.48894 |  0:10:04s\n",
      "epoch 78 | loss: 0.48535 |  0:10:12s\n",
      "epoch 79 | loss: 0.48607 |  0:10:20s\n",
      "epoch 80 | loss: 0.48998 |  0:10:28s\n",
      "epoch 81 | loss: 0.48636 |  0:10:35s\n",
      "epoch 82 | loss: 0.493   |  0:10:43s\n",
      "epoch 83 | loss: 0.50026 |  0:10:51s\n",
      "epoch 84 | loss: 0.49017 |  0:10:59s\n",
      "epoch 85 | loss: 0.49393 |  0:11:07s\n",
      "epoch 86 | loss: 0.48511 |  0:11:15s\n",
      "epoch 87 | loss: 0.48421 |  0:11:23s\n",
      "epoch 88 | loss: 0.48699 |  0:11:31s\n",
      "epoch 89 | loss: 0.49466 |  0:11:39s\n",
      "epoch 90 | loss: 0.49251 |  0:11:48s\n",
      "epoch 91 | loss: 0.48554 |  0:11:55s\n",
      "epoch 92 | loss: 0.4807  |  0:12:03s\n",
      "epoch 93 | loss: 0.48407 |  0:12:11s\n",
      "epoch 94 | loss: 0.47871 |  0:12:19s\n",
      "epoch 95 | loss: 0.48278 |  0:12:27s\n",
      "epoch 96 | loss: 0.49413 |  0:12:35s\n",
      "epoch 97 | loss: 0.4864  |  0:12:42s\n",
      "epoch 98 | loss: 0.48656 |  0:12:50s\n",
      "epoch 99 | loss: 0.47948 |  0:12:58s\n"
     ]
    }
   ],
   "source": [
    "from pytorch_tabnet.tab_model import TabNetClassifier\n",
    "\n",
    "tabnet_params = { 'mask_type': 'entmax','n_d': 8,  'gamma': 1.0, \"n_steps\":5,\n",
    "    'n_shared': 3, 'lambda_sparse':  0.0007131203334746554, 'device_name': 'cuda'}\n",
    "\n",
    "y_pred_tabnet_10000, y_pred_prob_tabnet_10000 = predict(classifier=TabNetClassifier,\n",
    "                                                              params=tabnet_params,\n",
    "       X_train=X_train_distance_band10000,   y_train=y_train_split, X_test=X_test_distance_band10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "043be7c1-780a-4e64-81e0-956881c6589d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mask_type': 'entmax',\n",
       " 'n_steps': 13,\n",
       " 'n_d': 8,\n",
       " 'gamma': 1.0,\n",
       " 'n_shared': 2,\n",
       " 'lambda_sparse': 2.450715921956413e-06,\n",
       " 'patienceScheduler': 3}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tabnet_model_KNN8 = joblib.load('study_tabnet_KNN8.pkl')\n",
    "tabnet_model_KNN8.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "80d2d1c0-c67b-45ef-99fb-d2443b5b8635",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  | loss: 1.40467 |  0:00:16s\n",
      "epoch 1  | loss: 0.89364 |  0:00:33s\n",
      "epoch 2  | loss: 0.78724 |  0:00:49s\n",
      "epoch 3  | loss: 0.76266 |  0:01:07s\n",
      "epoch 4  | loss: 0.71327 |  0:01:23s\n",
      "epoch 5  | loss: 0.61715 |  0:01:39s\n",
      "epoch 6  | loss: 0.58633 |  0:01:56s\n",
      "epoch 7  | loss: 0.56384 |  0:02:12s\n",
      "epoch 8  | loss: 0.55698 |  0:02:30s\n",
      "epoch 9  | loss: 0.54896 |  0:02:46s\n",
      "epoch 10 | loss: 0.54261 |  0:03:01s\n",
      "epoch 11 | loss: 0.54018 |  0:03:16s\n",
      "epoch 12 | loss: 0.53729 |  0:03:32s\n",
      "epoch 13 | loss: 0.5376  |  0:03:46s\n",
      "epoch 14 | loss: 0.53657 |  0:04:01s\n",
      "epoch 15 | loss: 0.53937 |  0:04:16s\n",
      "epoch 16 | loss: 0.53237 |  0:04:31s\n",
      "epoch 17 | loss: 0.53173 |  0:04:46s\n",
      "epoch 18 | loss: 0.5298  |  0:05:01s\n",
      "epoch 19 | loss: 0.52754 |  0:05:15s\n",
      "epoch 20 | loss: 0.53008 |  0:05:31s\n",
      "epoch 21 | loss: 0.52735 |  0:05:46s\n",
      "epoch 22 | loss: 0.52428 |  0:06:02s\n",
      "epoch 23 | loss: 0.52178 |  0:06:16s\n",
      "epoch 24 | loss: 0.52122 |  0:06:32s\n",
      "epoch 25 | loss: 0.51971 |  0:06:47s\n",
      "epoch 26 | loss: 0.52119 |  0:07:03s\n",
      "epoch 27 | loss: 0.51738 |  0:07:19s\n",
      "epoch 28 | loss: 0.51792 |  0:07:34s\n",
      "epoch 29 | loss: 0.51815 |  0:07:50s\n",
      "epoch 30 | loss: 0.51645 |  0:08:06s\n",
      "epoch 31 | loss: 0.5183  |  0:08:22s\n",
      "epoch 32 | loss: 0.51593 |  0:08:37s\n",
      "epoch 33 | loss: 0.51612 |  0:08:53s\n",
      "epoch 34 | loss: 0.51299 |  0:09:08s\n",
      "epoch 35 | loss: 0.51337 |  0:09:23s\n",
      "epoch 36 | loss: 0.51177 |  0:09:38s\n",
      "epoch 37 | loss: 0.5105  |  0:09:53s\n",
      "epoch 38 | loss: 0.50812 |  0:10:09s\n",
      "epoch 39 | loss: 0.50672 |  0:10:26s\n",
      "epoch 40 | loss: 0.50959 |  0:10:42s\n",
      "epoch 41 | loss: 0.50728 |  0:10:58s\n",
      "epoch 42 | loss: 0.50675 |  0:11:14s\n",
      "epoch 43 | loss: 0.50812 |  0:11:30s\n",
      "epoch 44 | loss: 0.51287 |  0:11:46s\n",
      "epoch 45 | loss: 0.50938 |  0:12:02s\n",
      "epoch 46 | loss: 0.50577 |  0:12:18s\n",
      "epoch 47 | loss: 0.50466 |  0:12:35s\n",
      "epoch 48 | loss: 0.50515 |  0:12:51s\n",
      "epoch 49 | loss: 0.5032  |  0:13:08s\n",
      "epoch 50 | loss: 0.50347 |  0:13:24s\n",
      "epoch 51 | loss: 0.50178 |  0:13:40s\n",
      "epoch 52 | loss: 0.5015  |  0:13:56s\n",
      "epoch 53 | loss: 0.50363 |  0:14:11s\n",
      "epoch 54 | loss: 0.50255 |  0:14:27s\n",
      "epoch 55 | loss: 0.49993 |  0:14:44s\n",
      "epoch 56 | loss: 0.49517 |  0:15:00s\n",
      "epoch 57 | loss: 0.49861 |  0:15:16s\n",
      "epoch 58 | loss: 0.49803 |  0:15:33s\n",
      "epoch 59 | loss: 0.49683 |  0:15:49s\n",
      "epoch 60 | loss: 0.49503 |  0:16:06s\n",
      "epoch 61 | loss: 0.49286 |  0:16:22s\n",
      "epoch 62 | loss: 0.49515 |  0:16:39s\n",
      "epoch 63 | loss: 0.50242 |  0:16:56s\n",
      "epoch 64 | loss: 0.50521 |  0:17:11s\n",
      "epoch 65 | loss: 0.50362 |  0:17:26s\n",
      "epoch 66 | loss: 0.50562 |  0:17:41s\n",
      "epoch 67 | loss: 0.50064 |  0:17:58s\n",
      "epoch 68 | loss: 0.49984 |  0:18:14s\n",
      "epoch 69 | loss: 0.49913 |  0:18:29s\n",
      "epoch 70 | loss: 0.49683 |  0:18:45s\n",
      "epoch 71 | loss: 0.49744 |  0:19:00s\n",
      "epoch 72 | loss: 0.49588 |  0:19:15s\n",
      "epoch 73 | loss: 0.49511 |  0:19:31s\n",
      "epoch 74 | loss: 0.49324 |  0:19:46s\n",
      "epoch 75 | loss: 0.50004 |  0:20:01s\n",
      "epoch 76 | loss: 0.50347 |  0:20:17s\n",
      "epoch 77 | loss: 0.50013 |  0:20:33s\n",
      "epoch 78 | loss: 0.49909 |  0:20:48s\n",
      "epoch 79 | loss: 0.49779 |  0:21:04s\n",
      "epoch 80 | loss: 0.50193 |  0:21:20s\n",
      "epoch 81 | loss: 0.49622 |  0:21:36s\n",
      "epoch 82 | loss: 0.49275 |  0:21:52s\n",
      "epoch 83 | loss: 0.49261 |  0:22:07s\n",
      "epoch 84 | loss: 0.49151 |  0:22:22s\n",
      "epoch 85 | loss: 0.49035 |  0:22:38s\n",
      "epoch 86 | loss: 0.49437 |  0:22:54s\n",
      "epoch 87 | loss: 0.49682 |  0:23:09s\n",
      "epoch 88 | loss: 0.49167 |  0:23:24s\n",
      "epoch 89 | loss: 0.49033 |  0:23:41s\n",
      "epoch 90 | loss: 0.49054 |  0:23:56s\n",
      "epoch 91 | loss: 0.48831 |  0:24:12s\n",
      "epoch 92 | loss: 0.48922 |  0:24:28s\n",
      "epoch 93 | loss: 0.48758 |  0:24:45s\n",
      "epoch 94 | loss: 0.48806 |  0:25:00s\n",
      "epoch 95 | loss: 0.48703 |  0:25:15s\n",
      "epoch 96 | loss: 0.48277 |  0:25:31s\n",
      "epoch 97 | loss: 0.48522 |  0:25:47s\n",
      "epoch 98 | loss: 0.48196 |  0:26:02s\n",
      "epoch 99 | loss: 0.48254 |  0:26:18s\n"
     ]
    }
   ],
   "source": [
    "tabnet_params = { 'mask_type': 'entmax','n_d': 8,  'gamma': 1.0, \"n_steps\":13,\n",
    "    'n_shared': 2, 'lambda_sparse': 2.450715921956413e-06, 'device_name': 'cuda'}\n",
    "\n",
    "y_pred_tabnet_KNN8, y_pred_prob_tabnet_KNN8= predict(classifier=TabNetClassifier,\n",
    "                                                              params=tabnet_params,\n",
    "       X_train=X_train_KNN8,   y_train=y_train_split, X_test=X_test_KNN8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "24653a7e-9e1d-4c88-9f15-e2119c663968",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mask_type': 'entmax',\n",
       " 'n_steps': 12,\n",
       " 'n_d': 6,\n",
       " 'gamma': 1.0,\n",
       " 'n_shared': 1,\n",
       " 'lambda_sparse': 0.0008914871643038309,\n",
       " 'patienceScheduler': 9}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tabnet_model_rook = joblib.load('study_tabnet_Rook.pkl')\n",
    "tabnet_model_rook.best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f152572f-65bf-4a1f-a141-326a5d7b35f8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0  | loss: 1.30714 |  0:00:12s\n",
      "epoch 1  | loss: 0.81082 |  0:00:24s\n",
      "epoch 2  | loss: 0.66744 |  0:00:36s\n",
      "epoch 3  | loss: 0.62348 |  0:00:48s\n",
      "epoch 4  | loss: 0.6022  |  0:01:01s\n",
      "epoch 5  | loss: 0.59091 |  0:01:13s\n",
      "epoch 6  | loss: 0.58215 |  0:01:25s\n",
      "epoch 7  | loss: 0.5698  |  0:01:36s\n",
      "epoch 8  | loss: 0.56759 |  0:01:48s\n",
      "epoch 9  | loss: 0.56404 |  0:01:59s\n",
      "epoch 10 | loss: 0.55948 |  0:02:12s\n",
      "epoch 11 | loss: 0.55896 |  0:02:24s\n",
      "epoch 12 | loss: 0.55248 |  0:02:36s\n",
      "epoch 13 | loss: 0.55216 |  0:02:48s\n",
      "epoch 14 | loss: 0.54816 |  0:03:00s\n",
      "epoch 15 | loss: 0.54876 |  0:03:11s\n",
      "epoch 16 | loss: 0.54824 |  0:03:23s\n",
      "epoch 17 | loss: 0.54436 |  0:03:35s\n",
      "epoch 18 | loss: 0.54378 |  0:03:46s\n",
      "epoch 19 | loss: 0.54237 |  0:03:58s\n",
      "epoch 20 | loss: 0.53696 |  0:04:10s\n",
      "epoch 21 | loss: 0.54087 |  0:04:23s\n",
      "epoch 22 | loss: 0.53748 |  0:04:35s\n",
      "epoch 23 | loss: 0.5331  |  0:04:47s\n",
      "epoch 24 | loss: 0.53567 |  0:05:00s\n",
      "epoch 25 | loss: 0.53521 |  0:05:12s\n",
      "epoch 26 | loss: 0.53194 |  0:05:24s\n",
      "epoch 27 | loss: 0.52874 |  0:05:37s\n",
      "epoch 28 | loss: 0.52559 |  0:05:48s\n",
      "epoch 29 | loss: 0.52588 |  0:06:00s\n",
      "epoch 30 | loss: 0.53006 |  0:06:12s\n",
      "epoch 31 | loss: 0.52711 |  0:06:24s\n",
      "epoch 32 | loss: 0.5224  |  0:06:38s\n",
      "epoch 33 | loss: 0.52156 |  0:06:51s\n",
      "epoch 34 | loss: 0.52288 |  0:07:05s\n",
      "epoch 35 | loss: 0.51847 |  0:07:16s\n",
      "epoch 36 | loss: 0.52068 |  0:07:28s\n",
      "epoch 37 | loss: 0.52118 |  0:07:40s\n",
      "epoch 38 | loss: 0.52475 |  0:07:51s\n",
      "epoch 39 | loss: 0.52133 |  0:08:03s\n",
      "epoch 40 | loss: 0.51866 |  0:08:14s\n",
      "epoch 41 | loss: 0.5192  |  0:08:26s\n",
      "epoch 42 | loss: 0.51783 |  0:08:39s\n",
      "epoch 43 | loss: 0.51406 |  0:08:51s\n",
      "epoch 44 | loss: 0.51345 |  0:09:03s\n",
      "epoch 45 | loss: 0.51331 |  0:09:14s\n",
      "epoch 46 | loss: 0.51258 |  0:09:26s\n",
      "epoch 47 | loss: 0.51156 |  0:09:38s\n",
      "epoch 48 | loss: 0.5104  |  0:09:49s\n",
      "epoch 49 | loss: 0.51212 |  0:10:01s\n",
      "epoch 50 | loss: 0.51063 |  0:10:12s\n",
      "epoch 51 | loss: 0.5122  |  0:10:24s\n",
      "epoch 52 | loss: 0.51401 |  0:10:36s\n",
      "epoch 53 | loss: 0.50934 |  0:10:48s\n",
      "epoch 54 | loss: 0.51109 |  0:11:00s\n",
      "epoch 55 | loss: 0.51008 |  0:11:12s\n",
      "epoch 56 | loss: 0.50296 |  0:11:23s\n",
      "epoch 57 | loss: 0.50636 |  0:11:35s\n",
      "epoch 58 | loss: 0.50902 |  0:11:46s\n",
      "epoch 59 | loss: 0.50579 |  0:11:58s\n",
      "epoch 60 | loss: 0.50671 |  0:12:10s\n",
      "epoch 61 | loss: 0.50259 |  0:12:22s\n",
      "epoch 62 | loss: 0.50128 |  0:12:33s\n",
      "epoch 63 | loss: 0.50201 |  0:12:45s\n",
      "epoch 64 | loss: 0.50263 |  0:12:56s\n",
      "epoch 65 | loss: 0.50108 |  0:13:08s\n",
      "epoch 66 | loss: 0.49818 |  0:13:19s\n",
      "epoch 67 | loss: 0.50103 |  0:13:31s\n",
      "epoch 68 | loss: 0.49976 |  0:13:42s\n",
      "epoch 69 | loss: 0.50009 |  0:13:54s\n",
      "epoch 70 | loss: 0.50002 |  0:14:06s\n",
      "epoch 71 | loss: 0.50088 |  0:14:18s\n",
      "epoch 72 | loss: 0.50207 |  0:14:30s\n",
      "epoch 73 | loss: 0.49983 |  0:14:42s\n",
      "epoch 74 | loss: 0.49933 |  0:14:54s\n",
      "epoch 75 | loss: 0.50057 |  0:15:06s\n",
      "epoch 76 | loss: 0.49826 |  0:15:17s\n",
      "epoch 77 | loss: 0.49854 |  0:15:29s\n",
      "epoch 78 | loss: 0.49576 |  0:15:40s\n",
      "epoch 79 | loss: 0.49384 |  0:15:52s\n",
      "epoch 80 | loss: 0.49527 |  0:16:04s\n",
      "epoch 81 | loss: 0.49739 |  0:16:15s\n",
      "epoch 82 | loss: 0.49062 |  0:16:27s\n",
      "epoch 83 | loss: 0.4939  |  0:16:39s\n",
      "epoch 84 | loss: 0.49617 |  0:16:51s\n",
      "epoch 85 | loss: 0.49529 |  0:17:03s\n",
      "epoch 86 | loss: 0.49136 |  0:17:15s\n",
      "epoch 87 | loss: 0.48914 |  0:17:26s\n",
      "epoch 88 | loss: 0.48918 |  0:17:38s\n",
      "epoch 89 | loss: 0.48783 |  0:17:50s\n",
      "epoch 90 | loss: 0.4928  |  0:18:02s\n",
      "epoch 91 | loss: 0.50523 |  0:18:15s\n",
      "epoch 92 | loss: 0.5005  |  0:18:26s\n",
      "epoch 93 | loss: 0.49762 |  0:18:39s\n",
      "epoch 94 | loss: 0.49451 |  0:18:52s\n",
      "epoch 95 | loss: 0.49038 |  0:19:03s\n",
      "epoch 96 | loss: 0.48786 |  0:19:15s\n",
      "epoch 97 | loss: 0.48949 |  0:19:26s\n",
      "epoch 98 | loss: 0.48765 |  0:19:38s\n",
      "epoch 99 | loss: 0.48683 |  0:19:50s\n"
     ]
    }
   ],
   "source": [
    "tabnet_params = { 'mask_type': 'entmax','n_d': 6,  'gamma': 1.0, \"n_steps\":12,\n",
    "    'n_shared': 1, 'lambda_sparse':0.0008914871643038309, 'device_name': 'cuda'}\n",
    "y_pred_tabnet_rook, y_pred_prob_tabnet_rook = predict(classifier=TabNetClassifier,\n",
    "                                                              params=tabnet_params,\n",
    "       X_train=X_train_rook,   y_train=y_train_split, X_test=X_test_rook)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15fbd275-7b15-4427-a385-1e28014fa9fc",
   "metadata": {},
   "source": [
    "<a id='section8'></a>\n",
    "# # Classification_report + metrics \n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "59db7355-102b-4f01-a20c-e59339973eac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>KNN Rook</th>\n",
       "      <th>KNN KNN8</th>\n",
       "      <th>KNN dist10000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision (Macro)</th>\n",
       "      <td>73.76%</td>\n",
       "      <td>73.73%</td>\n",
       "      <td>73.89%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall (Macro)</th>\n",
       "      <td>56.47%</td>\n",
       "      <td>57.21%</td>\n",
       "      <td>57.15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score (Macro)</th>\n",
       "      <td>57.96%</td>\n",
       "      <td>58.73%</td>\n",
       "      <td>58.71%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <td>56.47%</td>\n",
       "      <td>57.21%</td>\n",
       "      <td>57.15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Log Loss</th>\n",
       "      <td>0.9117440860445595</td>\n",
       "      <td>0.9028393960479246</td>\n",
       "      <td>0.8990080862772643</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             KNN Rook            KNN KNN8       KNN dist10000\n",
       "Precision (Macro)              73.76%              73.73%              73.89%\n",
       "Recall (Macro)                 56.47%              57.21%              57.15%\n",
       "F1-score (Macro)               57.96%              58.73%              58.71%\n",
       "Balanced Accuracy              56.47%              57.21%              57.15%\n",
       "Log Loss           0.9117440860445595  0.9028393960479246  0.8990080862772643"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_knn_rook = all_metrics(y_test, y_pred_KNN_rook, y_pred_prob_KNN_rook, \"KNN Rook\")\n",
    "metrics_knn_KNN8 = all_metrics(y_test, y_pred_knn_KNN8, y_pred_prob_knn_KNN8, \"KNN KNN8\")\n",
    "metrics_knn_dist10000 = all_metrics(y_test, y_pred_knn_10000, y_pred_prob_knn_10000, \"KNN dist10000\")\n",
    "\n",
    "combined_metrics_df_knn = pd.concat([metrics_knn_rook,  metrics_knn_KNN8, \n",
    "                                    metrics_knn_dist10000], axis=1)\n",
    "\n",
    "combined_metrics_df_knn.to_csv(\"combined_metrics_df_knn.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "34852b0e-6663-43cd-b093-8fd6f008a33b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TabNet Rook</th>\n",
       "      <th>TabNet KNN8</th>\n",
       "      <th>TabNet dist10000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision (Macro)</th>\n",
       "      <td>76.5%</td>\n",
       "      <td>77.67%</td>\n",
       "      <td>77.94%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall (Macro)</th>\n",
       "      <td>71.65%</td>\n",
       "      <td>73.18%</td>\n",
       "      <td>73.33%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score (Macro)</th>\n",
       "      <td>73.32%</td>\n",
       "      <td>74.72%</td>\n",
       "      <td>74.88%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <td>71.65%</td>\n",
       "      <td>73.18%</td>\n",
       "      <td>73.33%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Log Loss</th>\n",
       "      <td>0.628897</td>\n",
       "      <td>0.598817</td>\n",
       "      <td>0.593555</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  TabNet Rook TabNet KNN8 TabNet dist10000\n",
       "Precision (Macro)       76.5%      77.67%           77.94%\n",
       "Recall (Macro)         71.65%      73.18%           73.33%\n",
       "F1-score (Macro)       73.32%      74.72%           74.88%\n",
       "Balanced Accuracy      71.65%      73.18%           73.33%\n",
       "Log Loss             0.628897    0.598817         0.593555"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_tabnet_rook = all_metrics(y_test, y_pred_tabnet_rook, y_pred_prob_tabnet_rook, \"TabNet Rook\")\n",
    "metrics_tabnet_KNN8 = all_metrics(y_test, y_pred_tabnet_KNN8, y_pred_prob_tabnet_KNN8, \"TabNet KNN8\")\n",
    "metrics_tabnet_dist10000 = all_metrics(y_test,y_pred_tabnet_10000, y_pred_prob_tabnet_10000, \"TabNet dist10000\")\n",
    "\n",
    "combined_metrics_df_tabnet = pd.concat([metrics_tabnet_rook,metrics_tabnet_KNN8,\n",
    "                                    metrics_tabnet_dist10000], axis=1)\n",
    "\n",
    "combined_metrics_df_tabnet.to_csv(\"combined_metrics_df_tabnet.csv\")\n",
    "combined_metrics_df_tabnet\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "82d7ba97-156e-4672-a830-7e0cc462701f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Random Forest Rook</th>\n",
       "      <th>Random Forest KNN8</th>\n",
       "      <th>Random Forest dist10000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision (Macro)</th>\n",
       "      <td>78.03%</td>\n",
       "      <td>78.4%</td>\n",
       "      <td>78.18%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall (Macro)</th>\n",
       "      <td>73.74%</td>\n",
       "      <td>73.77%</td>\n",
       "      <td>73.47%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score (Macro)</th>\n",
       "      <td>74.95%</td>\n",
       "      <td>74.96%</td>\n",
       "      <td>74.86%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <td>73.74%</td>\n",
       "      <td>73.77%</td>\n",
       "      <td>73.47%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Log Loss</th>\n",
       "      <td>0.613405</td>\n",
       "      <td>0.591361</td>\n",
       "      <td>0.602151</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Random Forest Rook Random Forest KNN8  \\\n",
       "Precision (Macro)             78.03%              78.4%   \n",
       "Recall (Macro)                73.74%             73.77%   \n",
       "F1-score (Macro)              74.95%             74.96%   \n",
       "Balanced Accuracy             73.74%             73.77%   \n",
       "Log Loss                    0.613405           0.591361   \n",
       "\n",
       "                  Random Forest dist10000  \n",
       "Precision (Macro)                  78.18%  \n",
       "Recall (Macro)                     73.47%  \n",
       "F1-score (Macro)                   74.86%  \n",
       "Balanced Accuracy                  73.47%  \n",
       "Log Loss                         0.602151  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_rf_rook = all_metrics(y_test, y_pred_rf_rook, y_pred_prob_rf_rook, \"Random Forest Rook\")\n",
    "metrics_rf_KNN8 = all_metrics(y_test, y_pred_rf_KNN8,y_pred_prob_rf_KNN8, \"Random Forest KNN8\")\n",
    "metrics_rf_dist10000 = all_metrics(y_test, y_pred_rf_10000, y_pred_prob_rf_10000, \"Random Forest dist10000\")\n",
    "\n",
    "combined_metrics_df_rf = pd.concat([metrics_rf_rook, metrics_rf_KNN8,\n",
    "                                    metrics_rf_dist10000], axis=1)\n",
    "\n",
    "combined_metrics_df_rf.to_csv(\"combined_metrics_df_rf.csv\")\n",
    "combined_metrics_df_rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e2d3763f-bc19-4e08-9916-9adebaa9a6e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>XGBoost Rook</th>\n",
       "      <th>XGBoost KNN8</th>\n",
       "      <th>XGBoost dist10000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision (Macro)</th>\n",
       "      <td>79.27%</td>\n",
       "      <td>79.28%</td>\n",
       "      <td>79.39%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall (Macro)</th>\n",
       "      <td>75.58%</td>\n",
       "      <td>75.61%</td>\n",
       "      <td>75.74%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1-score (Macro)</th>\n",
       "      <td>76.8%</td>\n",
       "      <td>76.81%</td>\n",
       "      <td>77.0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <td>75.58%</td>\n",
       "      <td>75.61%</td>\n",
       "      <td>75.74%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Log Loss</th>\n",
       "      <td>0.543022</td>\n",
       "      <td>0.542075</td>\n",
       "      <td>0.539766</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  XGBoost Rook XGBoost KNN8 XGBoost dist10000\n",
       "Precision (Macro)       79.27%       79.28%            79.39%\n",
       "Recall (Macro)          75.58%       75.61%            75.74%\n",
       "F1-score (Macro)         76.8%       76.81%             77.0%\n",
       "Balanced Accuracy       75.58%       75.61%            75.74%\n",
       "Log Loss              0.543022     0.542075          0.539766"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_xgboost_rook = all_metrics(y_test, y_pred_xgb_rook, y_pred_prob_xgb_rook, \"XGBoost Rook\")\n",
    "metrics_xgboost_KNN8 = all_metrics(y_test, y_pred_xgb_KNN8, y_pred_prob_xgb_KNN8, \"XGBoost KNN8\")\n",
    "metrics_xgboost_dist10000 = all_metrics(y_test, y_pred_xgb_10000, y_pred_prob_xgb_10000, \"XGBoost dist10000\")\n",
    "\n",
    "combined_metrics_df_xgb = pd.concat([metrics_xgboost_rook,  metrics_xgboost_KNN8,\n",
    "                                    metrics_xgboost_dist10000], axis=1)\n",
    "\n",
    "\n",
    "combined_metrics_df_xgb.to_csv(\"combined_metrics_df_xgb.csv\")\n",
    "combined_metrics_df_xgb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "16f6c0e0-af90-42a2-97e8-ffd9fa572937",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification_report_xgb_Dist10000:\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "         Bicycle       0.74      0.76      0.75      9750\n",
      "    Car - Driver       0.78      0.84      0.81     11438\n",
      " Car - Passenger       0.70      0.46      0.55      3519\n",
      "            Foot       0.83      0.85      0.84      7082\n",
      "Public Transport       0.92      0.88      0.90      2493\n",
      "\n",
      "        accuracy                           0.78     34282\n",
      "       macro avg       0.79      0.76      0.77     34282\n",
      "    weighted avg       0.78      0.78      0.78     34282\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mapping = {'Bicycle':0, 'Car - Driver':1, 'Car - Passenger':2, 'Foot':3,\n",
    "       'Public Transport':4}\n",
    "\n",
    "print(\"classification_report_xgb_Dist10000:\")\n",
    "print(classification_report(y_test, y_pred_xgb_10000, target_names=list(mapping.keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "29a74b8e-dd44-4081-9a82-be0e17ecec26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6IAAAHqCAYAAAAebPOvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACvfElEQVR4nOzdd3gUVRfH8e8mpJFKgJCEktASeu+hg/QmKKJIL0qRjogi0kOR3hEERBARFREUQRCQjkjoXToJHQKkJ/v+wcvKEpQNbhISfh+ffR73zp07ZzZLsmfPnTsGo9FoRERERERERCSF2KR2ACIiIiIiIvJyUSIqIiIiIiIiKUqJqIiIiIiIiKQoJaIiIiIiIiKSopSIioiIiIiISIpSIioiIiIiIiIpSomoiIiIiIiIpCgloiIiIiIiIpKilIiKiIiIiIhIilIiKiLyEjh16hR16tTB3d0dg8HAqlWrrDr+uXPnMBgMLFq0yKrjpmXVq1enevXqqR1GkhkMBnr27Jls4z/tvTJs2DAMBkOyHVNERF48SkRFRFLImTNneOedd8iTJw+Ojo64ubkRFBTE1KlTiYyMTNZjt2vXjkOHDjF69GiWLFlCmTJlkvV4Kal9+/YYDAbc3Nye+jqeOnUKg8GAwWDg008/TfL4V65cYdiwYYSEhFghWrGWf/u5nDhxgr59+1KpUiUcHR0xGAycO3fuH8davXo1pUqVwtHRkVy5cvHJJ58QFxeXqN+dO3fo2rUrWbNmxdnZmRo1avDnn3+m2JgiIumJElERkRSwdu1aihYtyooVK2jcuDHTp08nODiYXLlyMXDgQHr37p1sx46MjGTnzp106tSJnj178vbbb5MjRw6rHsPPz4/IyEjatGlj1XEtlSFDBiIiIvjxxx8TbVu6dCmOjo7PPfaVK1cYPnx4khPR9evXs379+uc+7stkyJAhSf4y5t9+Ljt37mTatGncu3ePggUL/us4P//8M82aNcPDw4Pp06fTrFkzRo0axXvvvWfWLyEhgYYNG7Js2TJ69uzJ+PHjuXbtGtWrV+fUqVPJPqaISHqTIbUDEBFJ786ePUurVq3w8/Nj06ZN+Pj4mLb16NGD06dPs3bt2mQ7/vXr1wHw8PBItmMYDIb/lOz9Vw4ODgQFBfHVV1/RsmVLs23Lli2jYcOGfPvttykSS0REBBkzZsTe3j5FjpceZMiQgQwZrPeRpEmTJty5cwdXV1c+/fTTf/0SYcCAARQrVoz169ebYnBzc2PMmDH07t2bAgUKALBy5Up27NjBN998w2uvvQZAy5YtCQgI4JNPPmHZsmXJOqaISHqjiqiISDIbP3489+/fZ8GCBWZJ6CP58uUzq4jGxcUxcuRI8ubNi4ODA/7+/nz44YdER0eb7efv70+jRo3Ytm0b5cqVw9HRkTx58vDFF1+Y+gwbNgw/Pz8ABg4ciMFgwN/fH3g4pfXR/z/uadfrbdiwgcqVK+Ph4YGLiwuBgYF8+OGHpu3/dI3opk2bqFKlCs7Oznh4eNC0aVOOHTv21OOdPn2a9u3b4+Hhgbu7Ox06dCAiIuKfX9gnvPXWW/z888/cuXPH1LZ3715OnTrFW2+9laj/rVu3GDBgAEWLFsXFxQU3Nzfq16/PgQMHTH02b95M2bJlAejQoYNpiu+j86xevTpFihRh3759VK1alYwZM5pelyevEW3Xrh2Ojo6Jzr9u3bpkypSJK1eumNrOnDnDmTNn/vV8jUYjNWrUIGvWrFy7ds3UHhMTQ9GiRcmbNy8PHjwwO5cyZcrg6OhI3rx5mTt37r9em7l06VICAwNxdHSkdOnSbN269V/jeZo7d+7Qvn173N3d8fDwoF27dmY/n0eS+p571s/F09MTV1fXZ8Z39OhRjh49SteuXc0S4e7du2M0Glm5cqWpbeXKlWTLlo3mzZub2rJmzUrLli354YcfTP8+k2NMEZH0SImoiEgy+/HHH8mTJw+VKlWyqH/nzp0ZOnQopUqVYvLkyVSrVo3g4GBatWqVqO/p06d57bXXeOWVV5g4cSKZMmWiffv2HDlyBIDmzZszefJkAN58802WLFnClClTkhT/kSNHaNSoEdHR0YwYMYKJEyfSpEkTtm/f/q/7/frrr9StW5dr164xbNgw+vXrx44dOwgKCnrq9XotW7bk3r17BAcH07JlSxYtWsTw4cMtjrN58+YYDAa+++47U9uyZcsoUKAApUqVStT/r7/+YtWqVTRq1IhJkyYxcOBADh06RLVq1UxJYcGCBRkxYgQAXbt2ZcmSJSxZsoSqVauaxrl58yb169enRIkSTJkyhRo1ajw1vqlTp5I1a1batWtHfHw8AHPnzmX9+vVMnz4dX19fU99atWpRq1atfz1fg8HA559/TlRUFO+++66p/ZNPPuHIkSMsXLgQZ2dnAPbv30+9evW4efMmw4cPp1OnTowYMeIfF63asmULffr04e2332bEiBHcvHmTevXqcfjw4X+N6XFGo5GmTZuyZMkS3n77bUaNGsWlS5do167dM/d91nvOkp+LJfbv3w+Q6JppX19fcuTIYdr+qG+pUqWwsTH/6FSuXDkiIiI4efJkso0pIpIuGUVEJNncvXvXCBibNm1qUf+QkBAjYOzcubNZ+4ABA4yAcdOmTaY2Pz8/I2DcunWrqe3atWtGBwcHY//+/U1tZ8+eNQLGCRMmmI3Zrl07o5+fX6IYPvnkE+Pjfx4mT55sBIzXr1//x7gfHWPhwoWmthIlShi9vLyMN2/eNLUdOHDAaGNjY2zbtm2i43Xs2NFszFdffdWYOXPmfzzm4+fh7OxsNBqNxtdee81Yq1Yto9FoNMbHxxu9vb2Nw4cPf+prEBUVZYyPj090Hg4ODsYRI0aY2vbu3Zvo3B6pVq2aETDOmTPnqduqVatm1vbLL78YAeOoUaOMf/31l9HFxcXYrFmzRPv6+fk99WfzNHPnzjUCxi+//NK4a9cuo62trbFPnz5mfRo3bmzMmDGj8fLly6a2U6dOGTNkyGB88qMAYASMf/zxh6nt/PnzRkdHR+Orr75qUUxGo9G4atUqI2AcP368qS0uLs5YpUqVRK/n87zn/u3n8rgJEyYYAePZs2f/cduFCxcSbStbtqyxQoUKpufOzs6J3qNGo9G4du1aI2Bct25dso0pIpIeqSIqIpKMwsPDASyaJgjw008/AdCvXz+z9v79+wMkupa0UKFCVKlSxfQ8a9asBAYG8tdffz13zE96dG3pDz/8QEJCgkX7hIaGEhISQvv27fH09DS1FytWjFdeecV0no97vKoHUKVKFW7evGl6DS3x1ltvsXnzZsLCwti0aRNhYWFPnZYLD68rfVSJio+P5+bNm6YpoElZtdTBwYEOHTpY1LdOnTq88847jBgxgubNm+Po6MjcuXMT9Tt37ty/rvL6uK5du1K3bl3ee+892rRpQ968eRkzZoxpe3x8PL/++ivNmjUzq7rmy5eP+vXrP3XMihUrUrp0adPzXLly0bRpU3755RdTNfdZfvrpJzJkyEC3bt1Mbba2tokW7Hma53nPPY9HCyQ5ODgk2ubo6Gi2gFJkZOQ/9nt8rOQYU0QkPVIiKiKSjNzc3AC4d++eRf3Pnz+PjY0N+fLlM2v39vbGw8OD8+fPm7XnypUr0RiZMmXi9u3bzxlxYm+88QZBQUF07tyZbNmy0apVK1asWPGvCcKjOAMDAxNtK1iwIDdu3DC7fhESn0umTJkAknQuDRo0wNXVla+//pqlS5dStmzZRK/lIwkJCUyePJn8+fPj4OBAlixZyJo1KwcPHuTu3bsWHzN79uxJWpjo008/xdPTk5CQEKZNm4aXl5fF+/6TBQsWEBERwalTp1i0aBFOTk6mbdeuXSMyMvKpr8M/vTb58+dP1BYQEEBERIRp8atnOX/+PD4+Pri4uJi1P+098aTnec89j0ev09OuxYyKijJ7HZ2cnP6x3+NjJceYIiLpkRJREZFk5Obmhq+vb5KurQP+cQGZJ9na2j613Wg0Pvcxnqx4OTk5sXXrVn799VfatGnDwYMHeeONN3jllVcsro5Z4r+cyyMODg40b96cxYsX8/333/9jNRRgzJgx9OvXj6pVq/Lll1/yyy+/sGHDBgoXLpykhCepycL+/ftNiwsdOnQoSfv+k82bN5sSGmuNmZpS6j33aPGw0NDQRNtCQ0PNKsg+Pj7/2A8w9U2OMUVE0iMloiIiyaxRo0acOXOGnTt3PrOvn58fCQkJie4hePXqVe7cuWNaAdcaMmXK9NQVTJ+sugLY2NhQq1YtJk2axNGjRxk9ejSbNm3it99+e+rYj+I8ceJEom3Hjx8nS5YspoV0rO2tt95i//793Lt376kLPD2ycuVKatSowYIFC2jVqhV16tShdu3aiV4TS78UsMSDBw/o0KEDhQoVomvXrowfP569e/f+pzFDQ0N57733qFOnDo0aNWLAgAFmP0MvLy8cHR05ffp0on2f1gY89R6WJ0+eJGPGjGTNmtWiuPz8/AgNDeX+/ftm7U97TzzNs95z1vi5lChRAoA//vjDrP3KlStcunTJtP1R3z///DPRlxS7d+8mY8aMBAQEJNuYIiLpkRJREZFk9v777+Ps7Eznzp25evVqou1nzpxh6tSpwMOppUCilW0nTZoEQMOGDa0WV968ebl79y4HDx40tYWGhvL999+b9bt161aifR99mP6n20v4+PhQokQJFi9ebJbYHT58mPXr15vOMznUqFGDkSNHMmPGDLy9vf+xn62tbaJq6zfffMPly5fN2h4lzE9L2pNq0KBBXLhwgcWLFzNp0iT8/f1p165dotfRktu3PNKlSxcSEhJYsGAB8+bNI0OGDHTq1Ml0bra2ttSuXZtVq1aZ3SLm9OnT/Pzzz08dc+fOnWbXyV68eJEffviBOnXq/GPl+kkNGjQgLi6O2bNnm9ri4+OZPn36M/e15D1njZ9L4cKFKVCgAPPmzTOrtM6ePRuDwWC6tyfAa6+9xtWrV81WZb5x4wbffPMNjRs3Nl3rmRxjioikR9a7e7SIiDxV3rx5WbZsGW+88QYFCxakbdu2FClShJiYGNPN7Nu3bw9A8eLFadeuHfPmzePOnTtUq1aNPXv2sHjxYpo1a/aPtwZ5Hq1atWLQoEG8+uqr9OrVi4iICGbPnk1AQIBZEjJixAi2bt1Kw4YN8fPz49q1a8yaNYscOXJQuXLlfxx/woQJ1K9fn4oVK9KpUyciIyOZPn067u7uDBs2zGrn8SQbGxuGDBnyzH6NGjVixIgRdOjQgUqVKnHo0CGWLl1Knjx5zPrlzZsXDw8P5syZg6urK87OzpQvX57cuXMnKa5NmzYxa9YsPvnkE9PtZBYuXEj16tX5+OOPGT9+vKnvo1u3PGvBooULF7J27VoWLVpEjhw5AJg+fTpvv/02s2fPpnv37sDD+3SuX7+eoKAgunXrRnx8PDNmzKBIkSKEhIQkGrdIkSLUrVuXXr164eDgwKxZswCSdDudxo0bExQUxAcffMC5c+coVKgQ3333nUXX31rynvu3n8vdu3dNCe+jW77MmDEDDw8PPDw86Nmzp+lYEyZMoEmTJtSpU4dWrVpx+PBhZsyYQefOnSlYsKCp32uvvUaFChXo0KEDR48eJUuWLMyaNYv4+PhEr0tyjCkiku6k7qK9IiIvj5MnTxq7dOli9Pf3N9rb2xtdXV2NQUFBxunTpxujoqJM/WJjY43Dhw835s6d22hnZ2fMmTOncfDgwWZ9jMaHt/ho2LBhouM8eduQf7p9i9FoNK5fv95YpEgRo729vTEwMND45ZdfJrqVxsaNG41NmzY1+vr6Gu3t7Y2+vr7GN99803jy5MlEx3jyVhq//vqrMSgoyOjk5GR0c3MzNm7c2Hj06FGzPo+O9+StOhYuXPiPt9143OO3b/kn/3T7lv79+xt9fHyMTk5OxqCgIOPOnTufetuVH374wVioUCHT7U4enWe1atWMhQsXfuoxHx8nPDzc6OfnZyxVqpQxNjbWrF/fvn2NNjY2xp07d5raLLl9y8WLF43u7u7Gxo0bJ9r26quvGp2dnY1//fWXqW3jxo3GkiVLGu3t7Y158+Y1zp8/39i/f3+jo6Oj2b6AsUePHsYvv/zSmD9/fqODg4OxZMmSxt9+++1f43mamzdvGtu0aWN0c3Mzuru7G9u0aWPcv3//M2/fYsl7zmj855/Lo5/30x5Pe12///57Y4kSJYwODg7GHDlyGIcMGWKMiYlJ1O/WrVvGTp06GTNnzmzMmDGjsVq1asa9e/c+9dyTY0wRkfTEYDQmYRUIERERSTeaNWvGkSNHnnpNqIiISHLSNaIiIiIvgSfvSXnq1Cl++uknqlevnjoBiYjIS00VURERkZeAj48P7du3J0+ePJw/f57Zs2cTHR3N/v37n3rf0H8SHx//zHuJuri4JLp/qIiIyOO0WJGIiMhLoF69enz11VeEhYXh4OBAxYoVGTNmTJKSUHi4gu6zFmr65JNPknVBKhERSftUERURERGLRUVFsW3btn/tkydPnkSrD4uIiDxOiaiIiIiIiIikKC1WJCIiIiIiIilKiaiIiIiIiIikKC1WJMkiZ88fUjsEecH8PqxuaocgL5isbg6pHYK8QO5ExKZ2CPKCyexin9ohyAvGMY1kLk4le1p1vMj9M6w63otCFVERERERERFJUWnkewUREREREZE0wKBanyWUiIqIiIiIiFiLwZDaEaQJStdFREREREQkRakiKiIiIiIiYi2ammsRvUoiIiIiIiKSolQRFRERERERsRZdI2oRJaIiIiIiIiLWoqm5FtGrJCIiIiIiIilKFVERERERERFr0dRciygRFRERERERsRZNzbWIXiURERERERFJUaqIioiIiIiIWIum5lpEFVERERERERFJUaqIioiIiIiIWIuuEbWIElERERERERFr0dRciyhdFxERERERkRSliqiIiIiIiIi1aGquRZSIioiIiIiIWIum5lpE6bqIiIiIiIikKFVERURERERErEVTcy2iRFRERERERMRalIhaRK+SiIiIiIiIpChVREVERERERKzFRosVWUIVUREREREREUlRqoiKiIiIiIhYi64RtYgSUREREREREWvRfUQtonRdREREREREUpQqoiIiIiIiItaiqbkWUSIqIiIiIiJiLZqaaxGl6yIiIiIiIpKiVBEVERERERGxFk3NtYheJREREREREUlRSkRfMOfOncNgMBASEpIix6tevTp9+vRJkWOJiIiIiKR7BoN1H+mUpuamsPbt27N48WLTc09PT8qWLcv48eMpVqwYOXPmJDQ0lCxZsqRilPLIjuGvkDNzxkTti7eeZciKg2ZtX3SrQI3C2eg8bze/HAwztftmcmLMG8WoFJCFB9HxrNx9gbGrjxGfYEw0bpk8nnzTO4gTofeoN3az1c9H/rtDIfv4ZtkiTh0/xq2b1/kkeDKVqtYEIC4ulkXzZrB35zZCr1zC2dmVkmXL0+nd3mTO6mUaIzz8LrMmjWX39i0YbGyoXL0W3XoPwinjw/daTHQ00yaM4tSJo1w4f5bylaoybOyU1DhdeQ77/tjLF4sWcOzoEW5cv87EKTOoUau2afvNGzeYNvlTdu7czv179yhZugyDBg8hl5+/qc+333zNup/WcPzYUR48eMCW7XtwdXNLhbMRa4l48IDP585g25aN3Ll9i3wBBejZ7wMKFCpCXFwsn8+Zzu4dvxN6+TLOLi6UKluBLj36kOWx3x0fDXiPMyePc/v2LVxd3ShVtgJde/Y16yNp04rly1jx9VdcuXwZgLz58vNOt+5UrlINgJUrvubnn9Zw7OgRHjx4wO879+Km3wkvLk3NtYhepVRQr149QkNDCQ0NZePGjWTIkIFGjRoBYGtri7e3Nxky6DuCF0GjCVsoNXid6fHm9B0ArNl/2axf5xp5SJxWgo0BFnergF0GG5pN/J2+S/7k9fK5GNCwQKK+bk4ZmNKmFNtP3kiOUxEriYqMJE++QHr2H5xoW3RUFKdPHOet9l2Z+fnXDB0ziUsXzvHJoN5m/cYNH8z5s2cInjKHEeOncSjkT6aMH2HanpAQj72DA01ff4uSZcon+zmJdUVFRhIQUIAPPhqaaJvRaKRf7x5cunSJydNmsWzFd/j4+PJul45ERkT8PUZUFJWCqtCx8zspGboko0/HfMK+PTsZPGwMC5Z+R5nylRjYswvXr10lKiqKUyeO0abjO8z54muGj53MxQvnGDLgPbMxSpQuy9DRn7J4xY8MGzuZK5cvMmxwv1Q6I7Emr2ze9O47gK+++Y5lK76lXPkK9O7Zg9OnTwEQFRVJpaAqdOrybipHKmI9SkRTgYODA97e3nh7e1OiRAk++OADLl68yPXr1586NffIkSM0atQINzc3XF1dqVKlCmfOnGHr1q3Y2dkRFhZmNn6fPn2oUqWK6fn27dupXr06GTNmJFOmTNStW5fbt28/Nbbo6GgGDBhA9uzZcXZ2pnz58mzevDk5XoY04db9GK7fizY9ahXJxrnr99l16qapT6HsbnStmY8BX+5PtH/Vgl7k93al9+I/OXo5nM1Hr/Hp2uO0rZobO1vzqRbBrYqz6o9L7Dt7K9nPS55f2YqVad+1J0HVaiXa5uziytipc6lWqy45/fwpWKQYPfoN5tSJo1wLCwXgwrm/+GPXdvp+8AkFChejSPFSdO/7AVt+XcfN69cAcHTKSK+BQ2jQpAWenpodkdYEValKj159qFnrlUTbLpw/x6GDB/jw408oXKQo/rnz8OHHw4iOjmLdz2tN/Vq3aUeHzl0pWrx4SoYuySQ6Koqtv/3KOz37UbxkGbLnzEX7Lt3xzZGT1d99jYuLKxOmf0b12vXI5ZebQkWL02vAh5w8fpSr///dAfD6m20pVLQ43j6+FClWgjfbduLY4YPExcWm4tmJNVSvUZMqVavh5+ePv39u3uvdl4wZM3LwQAgAb7dtT6cuXSmm3wlpg6bmWkSJaCq7f/8+X375Jfny5SNz5syJtl++fJmqVavi4ODApk2b2LdvHx07diQuLo6qVauSJ08elixZYuofGxvL0qVL6dixIwAhISHUqlWLQoUKsXPnTrZt20bjxo2Jj49/ajw9e/Zk586dLF++nIMHD/L6669Tr149Tp06lTwvQBpiZ2ugedkcfL3zgqnN0c6W6e3LMGTFQa7fi060T+ncnhy/Es6Nx7ZtOXYNNyc7Anz+nlLTskIucmV2ZvLPJ5L3JCTFPbh/H4PBgLOrKwDHDh/AxdWVgIKFTX1KlSmPwcaG40cPpVaYkkJiYmIAsHdwMLXZ2Nhgb2dPyJ/7UissSWbx8fEkxMdj72Bv1u7g4MjhA4m/xAR4cP8eBoMBFxfXp24Pv3uXjb+spXDREmTIYGf1mCX1xMfH8/NPa4mMjKB48ZKpHY48D4ONdR/plOZ/poI1a9bg4uICwIMHD/Dx8WHNmjXY2CR+o82cORN3d3eWL1+Ond3DPzQBAQGm7Z06dWLhwoUMHDgQgB9//JGoqChatmwJwPjx4ylTpgyzZs0y7VO4cGGe5sKFCyxcuJALFy7g6+sLwIABA1i3bh0LFy5kzJgxVjj7tKtuMR/cnOz4ZvdFU9snLYqw7+wt1h8Ke+o+Wd0czJJQgOvh0aZtAP5ZnfmgSUFaTNn21OtGJe2KiY5mwewpVK9dH2fnh//mb928iYeHp1k/2wwZcHV149atm08bRtIR/9x58PbxZcaUSXw0dDhOGZ1Y+sVirl4N4/qN66kdniSTjM7OFCpanCWfzyWXfx4yeWZm0/qfOHr4AL45ciXqHxMdzbwZk6lZpz7O//+88Mi8GZNY9c1yoqIiKVSkGKMnzUyp05BkdurkCdq81YqYmGgyZszI5GkzyZsvX2qHJZJs0m+K/QKrUaMGISEhhISEsGfPHurWrUv9+vU5f/58or4hISFUqVLFlIQ+qX379pw+fZpdu3YBsGjRIlq2bImzs7Np/1q1Ek8hfJpDhw4RHx9PQEAALi4upseWLVs4c+bMP+4XHR1NeHi42cMYn/6mCbWq5MdvR69x9W4UAK8U9SYoIAvDVj5/FcvGADPal2bSTyc4e+2BtUKVF0BcXCyjPx4IRiPvDfwotcORF4SdnR2fTp7G+fPnqF65PJXKlmTv3t0EVa6KTTr+1ltg8LBgjEYjLRvVom6V0ny3Yhk169THxsZ82l1cXCzDPxqAEejz/seJxnnj7Q7MXbKC8dPmYmNjy9hhH2I06kvM9MDfPzcrvl3Fl1+t4PU33uTjDwdx5vTp1A5Lnoem5lpEFdFU4OzsTL7HvuGaP38+7u7ufPbZZ3Tu3Nmsr5OT07+O5eXlRePGjVm4cCG5c+fm559/Nrum81n7P+7+/fvY2tqyb98+bG1tzba5PPGN7OOCg4MZPny4WZtr2TdwL/emxcd+0WXP5ETlwKx0/WyPqa1SQBb8sjhzZEIDs75zO5djz5mbtJy6nevh0ZTwy2S2/VEl9Hp4NC6OGSjul4nCOdwZ+XpRAGwMBmxsDJyd2pjWM3eyQ4sXpTmPktCrV0MZP+0zUzUUwDNzZu7cMb8OOD4ujnv3wvH0TDw9X9KfQoWLsHzlKu7du0dcbCyZPD1p+1ZLChYqktqhSTLKniMnU+YsIjIygogHD8icJSsjPhqAj28OU5+4uFiGfziAq6FXmDhrQaJqKIC7RybcPTKRM5c/fv55eKPJKxw9fIDCRUuk4NlIcrCztyeXnx/w8PfEkcOHWPrlFwwdNuIZe8oLR18sWkSJ6AvAYDBgY2NDZGRkom3FihVj8eLFxMbG/mNVtHPnzrz55pvkyJGDvHnzEhQUZLb/xo0bEyWKT1OyZEni4+O5du2a2WJHzzJ48GD69TNfta/QoPUW758WtKyYixv3otl45Kqpbdb6UyzfYV7F/vWjmgz/9jC/Hn44VXff2Vu8VzeAzC723Lz/8NqwKgWyEh4Zy6mwe8TGJ1B79CazMdpWyU2lgCy8u2AvF25GIGnLoyT08sULjJ8+Hzd3D7PtBYsU5/69e5w6fpT8BQoBELJvD8aEBAoUKpoKEUtqcf3/dcMXzp/j6JHDdOvZK5UjkpTg5JQRJ6eM3Au/y95dO3inZ1/g7yT08sULTJq1APcnfnc8TcL/K6GxMelvFpJAQkICsf+/rlwkPVIimgqio6NNK93evn2bGTNmcP/+fRo3bpyob8+ePZk+fTqtWrVi8ODBuLu7s2vXLsqVK0dgYCAAdevWxc3NjVGjRjFihPm3ZoMHD6Zo0aJ0796dd999F3t7e3777Tdef/31RPcqDQgIoHXr1rRt25aJEydSsmRJrl+/zsaNGylWrBgNGzZ86vk4ODjg8NjCGwAG2/SzcILB8HAxoZW7L5pdw/loJd0nXbkdwcX/J5Bbj13jVNg9prYrzehVR/Byc2Bgo4J8sfUsMXEJAJwIvWe2/4370UTHJSRqlxdDZEQEVy79vWBV2JXLnDl5HFc3dzyzZGHkRwM4ffIYI8ZPJyEhgVs3H1a0Xd3csbOzI5d/HspUCGLKuOG8N3AI8XFxzJwcTLXa9czuNXr+7BniYmO5F36XiIgHnDl5HIC8AYlv/SMvloiIB1y88Pd75PLlS5w4fgw3d3d8fHzZ8Ms6Mnlmwtvbl9OnTjJh3Giq16xFxUqVTfvcuHGdmzdumMY5deokzs7OePv4WJSgyItn767tGI1Gcvr5c/niBeZOn0Quv9zUa9yMuLhYhn3Qj1MnjjFm4syn/u44dvggx48dpmjxUri4unHl8kUWzp2Bb46cFCqqlVTTuqmTJ1K5SlW8fXyIePCAn9au4Y+9e5g9bwEAN65f58ZjvxNOnzpJxozO+Pj44O7hkYqRy1OpImoRJaKpYN26dfj4+AAPvxEvUKAA33zzDdWrV+fcuXNmfTNnzsymTZsYOHAg1apVw9bWlhIlSphVPW1sbGjfvj1jxoyhbdu2ZvsHBASwfv16PvzwQ8qVK4eTkxPly5fnzTefPm124cKFjBo1iv79+3P58mWyZMlChQoVTPc5fRlVCcxKDs+MfL0r8TW8z5JghPazdzGmVXF+6F+FiOh4Vu65yKdrjydDpJISTh4/wvvv/T2Ffu70TwF4pX4T3u70Lru2bQage/uWZvuNnz6f4qXKAjDok2BmTgrmg15dMdjYULl6Lbr3+cCs/8cDenI17IrpefcObwDwy/YDVj8nsa6jRw7TtWM70/NJE8YC0LhJM4aPHsuNG9eYNGEsN2/eJEvWrDRq3JQu73YzG2PliuXMm/33IjSd278NwLCRY2jSrHkKnIVY24P79/hs1lRuXLuKq5s7VWrUplO3XmTIYEfYlcvs+H0zAF3avGa236RZn1OidFkcHB35/beNLJ43i8ioSDJnzkrZikG83aEr9vb2iQ8oacqtWzcZMngQ169fe7iyekAgs+ctoGKlh5/3vlmxnDmzZpj6d2jbGoARo4Jp+qp+J0jaZDDqCvd0oVOnTly/fp3Vq1endigA5Oz5Q2qHIC+Y34fVTe0Q5AXz6HppEYA7EZpeKuYyuyjBFnOOaaSE5tRktlXHi1zd7dmd0qA08uOUf3L37l0OHTrEsmXLXpgkVERERETkpaWpuRZRIprGNW3alD179vDuu+/yyiuvpHY4IiIiIiIiz6RENI17/FYtIiIiIiKSytLxvT+tSYmoiIiIiIiItWhqrkX0KomIiIiIiEiKUkVURERERETEWjQ11yJKREVERERERKzEoETUIpqaKyIiIiIiIilKFVERERERERErUUXUMqqIioiIiIiISIpSRVRERERERMRaVBC1iBJRERERERERK9HUXMtoaq6IiIiIiIikKFVERURERERErEQVUcsoERUREREREbESJaKW0dRcERERERERSVGqiIqIiIiIiFiJKqKWUUVUREREREREUpQSUREREREREWsxWPlhofj4eD7++GNy586Nk5MTefPmZeTIkRiNRlMfo9HI0KFD8fHxwcnJidq1a3Pq1CmzcW7dukXr1q1xc3PDw8ODTp06cf/+fbM+Bw8epEqVKjg6OpIzZ07Gjx9veaD/p0RURERERETESgwGg1Uflho3bhyzZ89mxowZHDt2jHHjxjF+/HimT59u6jN+/HimTZvGnDlz2L17N87OztStW5eoqChTn9atW3PkyBE2bNjAmjVr2Lp1K127djVtDw8Pp06dOvj5+bFv3z4mTJjAsGHDmDdvXpJeJ10jKiIiIiIiksbt2LGDpk2b0rBhQwD8/f356quv2LNnD/CwGjplyhSGDBlC06ZNAfjiiy/Ili0bq1atolWrVhw7dox169axd+9eypQpA8D06dNp0KABn376Kb6+vixdupSYmBg+//xz7O3tKVy4MCEhIUyaNMksYX0WVURFRERERESsxNoV0ejoaMLDw80e0dHRiY5bqVIlNm7cyMmTJwE4cOAA27Zto379+gCcPXuWsLAwateubdrH3d2d8uXLs3PnTgB27tyJh4eHKQkFqF27NjY2NuzevdvUp2rVqtjb25v61K1blxMnTnD79m2LXycloiIiIiIiIlZi7UQ0ODgYd3d3s0dwcHCi437wwQe0atWKAgUKYGdnR8mSJenTpw+tW7cGICwsDIBs2bKZ7ZctWzbTtrCwMLy8vMy2Z8iQAU9PT7M+Txvj8WNYQlNzRUREREREXlCDBw+mX79+Zm0ODg6J+q1YsYKlS5eybNky03TZPn364OvrS7t27VIqXIspERUREREREbESa99H1MHB4amJ55MGDhxoqooCFC1alPPnzxMcHEy7du3w9vYG4OrVq/j4+Jj2u3r1KiVKlADA29uba9eumY0bFxfHrVu3TPt7e3tz9epVsz6Pnj/qYwlNzRUREREREbGWVLp9S0REBDY25umdra0tCQkJAOTOnRtvb282btxo2h4eHs7u3bupWLEiABUrVuTOnTvs27fP1GfTpk0kJCRQvnx5U5+tW7cSGxtr6rNhwwYCAwPJlCmTxfEqERUREREREUnjGjduzOjRo1m7di3nzp3j+++/Z9KkSbz66qvAw0ptnz59GDVqFKtXr+bQoUO0bdsWX19fmjVrBkDBggWpV68eXbp0Yc+ePWzfvp2ePXvSqlUrfH19AXjrrbewt7enU6dOHDlyhK+//pqpU6cmmj78LJqaKyIiIiIiYiXWnpprqenTp/Pxxx/TvXt3rl27hq+vL++88w5Dhw419Xn//fd58OABXbt25c6dO1SuXJl169bh6Oho6rN06VJ69uxJrVq1sLGxoUWLFkybNs203d3dnfXr19OjRw9Kly5NlixZGDp0aJJu3QJgMBqNxv9+2iLmcvb8IbVDkBfM78PqpnYI8oLJ6vbs613k5XEnIvbZneSlktnF/tmd5KXimEZKaFnaL7fqeDcWtbLqeC+KNPLjFBERERERefGlVkU0rVEiKiIiIiIiYiVKRC2jxYpEREREREQkRakiKiIiIiIiYi0qiFpEiaiIiIiIiIiVaGquZTQ1V0RERERERFKUKqKSLHSrDnlSwVcGpHYI8oIJ2zHt2Z3kpeHuZJfaIcgLRjcYlLRKFVHLKBEVERERERGxEiWiltHUXBEREREREUlRqoiKiIiIiIhYiSqillFFVERERERERFKUKqIiIiIiIiLWooKoRZSIioiIiIiIWImm5lpGU3NFREREREQkRakiKiIiIiIiYiWqiFpGiaiIiIiIiIiVKBG1jKbmioiIiIiISIpSRVRERERERMRaVBC1iCqiIiIiIiIikqJUERUREREREbESXSNqGSWiIiIiIiIiVqJE1DKamisiIiIiIiIpShVRERERERERK1FF1DJKREVERERERKxEiahlNDVXREREREREUpQqoiIiIiIiItaigqhFlIiKiIiIiIhYiabmWkZTc0VERERERCRFqSIqIiIiIiJiJaqIWkYVUREREREREUlRqoiKiIiIiIhYiQqillEiKiIiIiIiYiWammsZTc0VERERERGRFKWKqIiIiIiIiJWoIGoZJaIiIiIiIiJWoqm5ltHUXBEREREREUlRqoiKiIiIiIhYiQqillEiKiIiIiIiYiU2NspELaGpuSIiIiIiIpKiVBEVERERERGxEk3NtYwqounIsGHDKFGiRGqHISIiIiIi8q9e2opoWFgYo0ePZu3atVy+fBkvLy9KlChBnz59qFWrVqrE5O/vz/nz5wFwdHQkW7ZslCtXjnfffZeaNWs+c/8BAwbw3nvvJXeYL5VDIfv4ZtkiTh0/xq2b1/kkeDKVqv79s9i2+VfWrvqGUyeOcS/8LrMWfk3egAKm7WGhl2n3WoOnjv3RyAlUrVmH8Lt3GDt8MGdPn+Je+B3cM3lSsXJ1OrzbC2dnl2Q/R0k6l4wOfNK9EU1qFidrJhcOnLjEgPEr2Xf0gqlPYO5sjOrdjCql8pEhgw3H/wrjzQHzuRh2G4COzYN4o34ZShTIgZuLE95VBnL3fuRTj2dvl4GtSwZQPDAH5d8I5uDJyylynvJ8Vq74iu++WU7olYc/p9x589G5a3cqVa5q6nPwwH5mz5jKkUMHsbW1IX9gAabNmo+joyP79u6hW5d2Tx170ZcrKFSkaIqch1jP5/Pn8tvGDZw7+xcODo4UK1GSXn364587j6nPdyu/Zt1Pazh+7CgPHjxg87Y9uLq5PXW8mJgY2rVuyckTx1m24nsCCxRMqVORZFK/Tk3T74zHtWz1Fh8O+YTo6GgmThjLLz//RExMDJWCKvPhkE/InCVLKkQrz6Lbt1jmpUxEz507R1BQEB4eHkyYMIGiRYsSGxvLL7/8Qo8ePTh+/PhzjRsfH4/BYMDG5vkLzSNGjKBLly7ExMRw7tw5vvzyS2rXrs3IkSP56KOPnrqP0WgkPj4eFxcXXFySN3GJiYnB3t4+WY/xIomKjCRPvkDqNmzGiA/7Jd4eFUnhYiWpWrMuU8YNT7Q9q5c3X63eaNb20w8rWblsMWUrVAbAYLChYpUatO/SE/dMmbhy6SIzJo7h3oRRDB42NnlOTP6T2UPfolA+XzoOWUzo9bu82aAca+e8R6kWo7hy/S65c2Rh4+f9WLxqB6NmryX8QRSF8voQFR1rGiOjox0bdhxlw46jjOzV9F+PN6ZPU0Kv36V4YI7kPjWxgmzZvOnRqx85c/lhxMja1T8woE9Pliz/lrz58nPwwH569+hK+45dGTDoIzJkyMDJE8dNfzuKlSjBT79uNRtz7sxp7N2zi4KFi6TGKcl/9Ocfe3m91VsULlyU+Ph4ZkybTI93O7Py+zU4ZcwIQFRkFBWDqlAxqAozpk761/GmTppA1qxenDzxfJ9X5MWzdPlKEhLiTc9PnzrFu1068EqdegB8Om4Mv2/dwoRJU3BxcWXsmJH069OTxV8uT62Q5V8oD7XMSzk1t3v37hgMBvbs2UOLFi0ICAigcOHC9OvXj127dpn6TZo0iaJFi+Ls7EzOnDnp3r079+/fN21ftGgRHh4erF69mkKFCuHg4MCFCxeedkiLubq64u3tTa5cuahatSrz5s3j448/ZujQoZw4cQKAzZs3YzAY+PnnnyldujQODg5s27bNbGru+vXrcXR05M6dO2bj9+7d26y6um3bNqpUqYKTkxM5c+akV69ePHjwwLTd39+fkSNH0rZtW9zc3Ojatet/Or+0pmzFyrTv2pOgak+vkteu15i3O75LybLln7rd1tYWz8xZzB47tm6iaq06pg8frm5uNH61JQEFC5PN25eSZcrTuHlLDh/4M9nOS56fo4MdzWqV4KMpq9j+5xn+uniD0XN/4szF63R5vQoAw3s25pdtR/ho6g8cOHGJs5dusHbLIa7f/vv3x4xlm/l04QZ2Hzz3r8erE1SIWhUKMnjy98l5WmJFVarVIKhKNXL5+ePnl5vu7/UhY8aMHD50AIApn47ljTffpl3HLuTNlx8//9y8Ure+6Us+Ozt7smTJanp4uHuwdfMmGjd9Vd+yp1Ez5synSdPm5M2Xn4DAAgwfGUxY6BWOHT1i6vNWm3Z06NSVosWK/+tY23/fyq6d2+nT//3kDltSkKenp9m/+61bfiNnzlyUKVuOe/fu8f1339L//Q8oV74ihQoXYfjIMRwI2c/BAyGpHbrIc3vpEtFbt26xbt06evTogbOzc6LtHh4epv+3sbFh2rRpHDlyhMWLF7Np0ybef9/8F39ERATjxo1j/vz5HDlyBC8vL6vH3Lt3b4xGIz/88INZ+wcffMDYsWM5duwYxYoVM9tWq1YtPDw8+Pbbb01t8fHxfP3117Ru3RqAM2fOUK9ePVq0aMHBgwf5+uuv2bZtGz179jQb69NPP6V48eLs37+fjz/+2Orn9zI5dfwoZ06doG6jV/+xz83r19i+ZRPFSpROwcjEUhlsbciQwZaomFiz9qjoWCqVzIvBYKBe5cKcunCN1TN7cH5jMFu/GEDj6sX+YcR/5uXpyqyP36TTx18QERljrVOQFBQfH8/6dWuJjIygaLES3Lp1k8OHDpLJMzOd2r5JvZqVeadTG0L27/vHMbZu+Y27d+/QqGnzFIxcktP9+/cAcHN3T9J+N2/eYNTwjxk5ZhyOjo7JEZq8AGJjY/hpzWqavtoCg8HAsaOHiYuLpXyFSqY+ufPkxcfHlwNKRF9IBoPBqo/06qVLRE+fPo3RaKRAgQLP7NunTx9q1KiBv78/NWvWZNSoUaxYscKsT2xsLLNmzaJSpUoEBgaS8f9VLmvy9PTEy8uLc+fOmbWPGDGCV155hbx58+Lp6Wm2zdbWllatWrFs2TJT28aNG7lz5w4tWrQAIDg4mNatW9OnTx/y589PpUqVmDZtGl988QVRUVGm/WrWrEn//v3JmzcvefPmtfr5vUzWrfmeXP55KFy0RKJtwZ8MoknN8rzV7BUyZnSm7wfDUjw+ebb7EdHsOvAXg7vUxyerOzY2Blo1KEv5YrnxzuKGl6cLrs6ODOjwCht2HKVxtxms/u0Ayyd2pnLpfEk61rwRb/PZym38efS/zbSQlHf61EmqVSxN5XLFGTtqOOMnTSdP3nxcvnQRgM/mzKBZ89eZOmsegQUK0aNrBy6cP/fUsVZ/v5IKFYPIls07Bc9AkktCQgKfjh9D8ZKlyJc/wOL9jEYjw4YMpkXLVhQqrOuE07NNG3/l3r17NGn28EvrGzduYGdnh9sT1wx7Zs7MzRvXUyNEeQYlopZ56RJRo9Focd9ff/2VWrVqkT17dlxdXWnTpg03b94kIiLC1Mfe3j5RNfJJhQsXNl2/Wb9+/eeO+8k3YpkyZf51n9atW7N582auXLkCwNKlS2nYsKGp6nvgwAEWLVpkis3FxYW6deuSkJDA2bNnLT5OdHQ04eHhZo/o6OjnOMv0LTo6it82/EzdRs2euv2dXgOZsXA5w8ZO5crli8yd/mnKBigW6zjkCwwG+Gv9aO7unkKPN6uxYt0fJCQYTdf5rdl8iOlLf+Pgyct8unADP/1+hC6vVbb4GN3frIZrRkcmfL4+uU5DkpGfvz9ffv0dny/5mhYtWzF86GD+OnMaY8LDv0HNW7xB42bNCSxQiH4DB+Pnn5sff/gu0ThXr4axa+d2mrz6WkqfgiSTsaNHcOb0KYLH/ft1oE9avmwJDyIe0KHTy3WJzMto1XffElS5Kl5e2VI7FJFk9dItVpQ/f34MBsMzFyQ6d+4cjRo1olu3bowePRpPT0+2bdtGp06diImJMVU+nZycnvlNxU8//URsbKypf1LdvHmT69evkzt3brP2p00tflzZsmXJmzcvy5cvp1u3bnz//fcsWrTItP3+/fu888479OrVK9G+uXLlsvg4wcHBDB9uvlBP74Ef0ef9If+638vm9982EB0VSe16jZ+6/dE1pLn8cuPq5kb/7h14q31XMmfJmsKRyrOcvXSDOp2nktHRHjcXR8JuhLNkbAfOXr7Bjdv3iY2N59hfoWb7nPgrjEol8/zDiIlVLxtA+WK5ubt7iln79qXvs/znP+gydIk1TkWSiZ2dPTlz+QFQsFBhjh45xNfLltC2YxcAcj8xu8Q/dx7CQkMTjbPmh+9wd/egarUayR+0JLtxY0awbetmPlv4Jdm8k1bh3rtnN4cOhFCxjPmX323efI16DRoxYvQ4a4YqqeTKlcvs3rWDiVOmm9qyZMlCbGws4eHhZlXRWzdv6jPCCyodFzGt6qVLRD09Palbty4zZ86kV69eiZKsO3fu4OHhwb59+0hISGDixImmCseT03It5efn959injp1KjY2NjRr1izJ+7Zu3ZqlS5eSI0cObGxsaNiwoWlbqVKlOHr0KPnyJW264JMGDx5Mv37mK8qG3rO88vyy+GXNKipUro5HJs9n9n1UuY+N1XWBL7KIqBgiomLwcHWidqWCfDTlB2Lj4tl39DwBfubfZOf38+JC6G2Lx+4/fiXDZq4xPffJ6s6a2T1p88FC9h46Z61TkBSSkGAkJiYGX9/sZM3qxflzZ822Xzh/nkpBVczajEYjP/7wPQ0aNyWDnV1KhitWZjQaGR88kt82/cq8BV+QPUfSV8Ae+MFHdO/Z2/T8+vVr9Hy3M8HjJ1Gk6L8vcCRpxw/ff4enZ2aqVK1uaitYqAgZMtixZ/dOar9SF4BzZ/8iNPQKxYuXSJ1ARazgpUtEAWbOnElQUBDlypVjxIgRFCtWjLi4ODZs2MDs2bM5duwY+fLlIzY2lunTp9O4cWO2b9/OnDlzkj22e/fuERYWRmxsLGfPnuXLL79k/vz5BAcHP1fC2Lp1a4YNG8bo0aN57bXXcHBwMG0bNGgQFSpUoGfPnnTu3BlnZ2eOHj3Khg0bmDFjhsXHcHBwMBsX4FZM1D/0TlsiIyK4cunv6/PCrlzmzMnjuLq54+XtQ3j4Xa6HhZqu0bh44RwAmf5f3Xzk8qULHArZx8hPZyY6xp4dv3P79k0CCxbG0Skj58+eYf7MyRQuVgJvn+zJe4LyXGpXLIjBACfPXSNvzqyM6duMk2ev8sXqnQBMXvwrS8Z1ZNufp9nyx0nqVCpEg6pFqNtlqmmMbJldyZbZjby5Hr5PiuT35d6DKC6G3eZ2eITpfqOP3I94ON39r4vXuXztTsqcqDyXmdMmUTGoCt7evkREPOCXn9fw5x97mDbrMwwGA2+368i8OTPIH1CAgMACrP1xFefP/cXYT6eYjbN3zy6uXL5EU03LTfPGjh7Bup/XMGnqTDI6O3Pj/38zXFxcTYsO3bhxnZs3bnDx/6vvnz51kozOznj7+ODu7oGPj6/ZmI9mZuXImSvJ1VV5MSUkJLB61Xc0btqMDBn+/oju6urKq81bMHH8WNzd3XF2dmHsmFEUK16SYkpEX0jp+bpOa3opE9E8efLw559/Mnr0aPr3709oaChZs2aldOnSzJ49G4DixYszadIkxo0bx+DBg6latSrBwcG0bds2WWMbOnQoQ4cOxd7eHm9vbypUqMDGjRupUeP5pmXly5ePcuXKsWfPHqZMmWK2rVixYmzZsoWPPvqIKlWqYDQayZs3L2+88YYVziR9OHn8CO+/19n0/NF1m6/Ub8KAISPZ9ftmJo4Zatoe/MkgAN7u+C5tOnUztf+yZhVZvLJRulzFRMewd3Dg59XfMXfap8TGxJA1WzaCqtXijbc7JtNZyX/l7uLIiPeakD2bB7fuRvDDxhA+mfkjcXEJAKz+7SDvjV7OwI51mPj+a5w8f403B85nR8hfpjE6v1aFIe82MD3/9fO+AHQZuoQvf9ydsickVnXr1k2GD/mAGzeu4+LiSr6AAKbN+ozyFYMAePPtdsTExDD507GE371L/oBAps9ZQI6cuczGWf39txQrXhL/3JZP6ZYX08oVXwHQtaP5Z4hPRo6hyf9XQ/52xXLmzfn7y8rOHd5O1EfSt107dxAaeoVmr7ZItG3AoA8x2NjQv08vYmJjqFSpMh9+/EkqRCmWUB5qGYMxKav3iFjo3I30UREV6yn4yoDUDkFeMGE7pqV2CPICsbXRJzcxp/eEPMkpjVylUGrEJquO9+fQmlYd70XxUlZERUREREREkoOm5lpGiaiIiIiIiIiVKA+1zEt3H1ERERERERFJXaqIioiIiIiIWImm5lpGiaiIiIiIiIiVKA+1jKbmioiIiIiISIpSRVRERERERMRKNDXXMqqIioiIiIiISIpSRVRERERERMRKVBC1jBJRERERERERK9HUXMtoaq6IiIiIiIikKFVERURERERErEQFUcsoERUREREREbESTc21jKbmioiIiIiISIpSRVRERERERMRKVBC1jCqiIiIiIiIikqJUERUREREREbESXSNqGSWiIiIiIiIiVqJE1DKamisiIiIiIiIpShVRERERERERK1FB1DJKREVERERERKxEU3Mto6m5IiIiIiIi6cDly5d5++23yZw5M05OThQtWpQ//vjDtN1oNDJ06FB8fHxwcnKidu3anDp1ymyMW7du0bp1a9zc3PDw8KBTp07cv3/frM/BgwepUqUKjo6O5MyZk/Hjxyc5ViWiIiIiIiIiVmIwWPdhqdu3bxMUFISdnR0///wzR48eZeLEiWTKlMnUZ/z48UybNo05c+awe/dunJ2dqVu3LlFRUaY+rVu35siRI2zYsIE1a9awdetWunbtatoeHh5OnTp18PPzY9++fUyYMIFhw4Yxb968pL1ORqPRmKQ9RCxw7kbUszvJS6XgKwNSOwR5wYTtmJbaIcgLxNZGU9nEnN4T8iQnu9SOwDI1p+206nibelW0qN8HH3zA9u3b+f3335+63Wg04uvrS//+/Rkw4OHnsrt375ItWzYWLVpEq1atOHbsGIUKFWLv3r2UKVMGgHXr1tGgQQMuXbqEr68vs2fP5qOPPiIsLAx7e3vTsVetWsXx48ctPi9VREVERERERF5Q0dHRhIeHmz2io6MT9Vu9ejVlypTh9ddfx8vLi5IlS/LZZ5+Ztp89e5awsDBq165tanN3d6d8+fLs3Pkwed65cyceHh6mJBSgdu3a2NjYsHv3blOfqlWrmpJQgLp163LixAlu375t8XkpERUREREREbESa0/NDQ4Oxt3d3ewRHByc6Lh//fUXs2fPJn/+/Pzyyy9069aNXr16sXjxYgDCwsIAyJYtm9l+2bJlM20LCwvDy8vLbHuGDBnw9PQ06/O0MR4/hiW0aq6IiIiIiMgLavDgwfTr18+szcHBIVG/hIQEypQpw5gxYwAoWbIkhw8fZs6cObRr1y5FYk0KVURFRERERESsxMZgsOrDwcEBNzc3s8fTElEfHx8KFSpk1lawYEEuXLgAgLe3NwBXr14163P16lXTNm9vb65du2a2PS4ujlu3bpn1edoYjx/DotfJ4p4iIiIiIiLyr1Jr1dygoCBOnDhh1nby5En8/PwAyJ07N97e3mzcuNG0PTw8nN27d1Ox4sMFkSpWrMidO3fYt2+fqc+mTZtISEigfPnypj5bt24lNjbW1GfDhg0EBgaardD7LEpERURERERE0ri+ffuya9cuxowZw+nTp1m2bBnz5s2jR48eABgMBvr06cOoUaNYvXo1hw4dom3btvj6+tKsWTPgYQW1Xr16dOnShT179rB9+3Z69uxJq1at8PX1BeCtt97C3t6eTp06ceTIEb7++mumTp2aaPrwsyT5GtGLFy9iMBjIkSMHAHv27GHZsmUUKlTI7P4yIiIiIiIiLxtDUsqYVlS2bFm+//57Bg8ezIgRI8idOzdTpkyhdevWpj7vv/8+Dx48oGvXrty5c4fKlSuzbt06HB0dTX2WLl1Kz549qVWrFjY2NrRo0YJp0/6+5Zq7uzvr16+nR48elC5dmixZsjB06NAk54JJvo9olSpV6Nq1K23atCEsLIzAwEAKFy7MqVOneO+99xg6dGiSApD0SfcRlSfpPqLyJN1HVB6ne0bKk/SekCellfuI1p+926rj/dytvFXHe1EkeWru4cOHKVeuHAArVqygSJEi7Nixg6VLl7Jo0SJrxyciIiIiIiLpTJKn5sbGxppWafr1119p0qQJAAUKFCA0NNS60YmIiIiIiKQhqTU1N61JckW0cOHCzJkzh99//50NGzZQr149AK5cuULmzJmtHqCIiIiIiEhakVqr5qY1Sa6Ijhs3jldffZUJEybQrl07ihcvDsDq1atNU3ZFXByT/NaSdO7Yr5+mdgjygrlyJzK1Q5AXiH8W59QOQUREUlCSs4Xq1atz48YNwsPDze4T07VrVzJmzGjV4ERERERERNISA+m4jGlFz3UfUaPRyL59+5g7dy737t0DwN7eXomoiIiIiIiIPFOSK6Lnz5+nXr16XLhwgejoaF555RVcXV0ZN24c0dHRzJkzJzniFBEREREReeHpzkOWSXJFtHfv3pQpU4bbt2/j5ORkan/11VfZuHGjVYMTERERERFJSwwGg1Uf6VWSK6K///47O3bswN7e3qzd39+fy5cvWy0wERERERERSZ+SnIgmJCQQHx+fqP3SpUu4urpaJSgREREREZG0KB0XMa0qyVNz69Spw5QpU0zPDQYD9+/f55NPPqFBgwbWjE1ERERERCRNsTEYrPpIr5JcEZ04cSJ169alUKFCREVF8dZbb3Hq1CmyZMnCV199lRwxioiIiIiISDqS5EQ0R44cHDhwgOXLl3Pw4EHu379Pp06daN26tdniRSIiIiIiIi+bdFzEtKokJ6IAGTJk4O2337Z2LCIiIiIiIvISSHIi+sUXX/zr9rZt2z53MCIiIiIiImlZer7lijUlORHt3bu32fPY2FgiIiKwt7cnY8aMSkRFREREROSlpTzUMkleNff27dtmj/v373PixAkqV66sxYpERERERETkmZ7rGtEn5c+fn7Fjx/L2229z/PhxawwpIiIiIiKS5qTnW65Yk1USUXi4gNGVK1esNZyIiIiIiEiaozTUMklORFevXm323Gg0EhoayowZMwgKCrJaYCIiIiIiIpI+JTkRbdasmdlzg8FA1qxZqVmzJhMnTrRWXCIiIiIiImmOVs21TJIT0YSEhOSIQ0REREREJM2zUR5qkSSvmisiIiIiIiLyX1hUEe3Xr5/FA06aNOm5gxEREREREUnLNDXXMhYlovv377doML3oIiIiIiIi8iwWJaK//fZbcschIiIiIiKS5qk2Zxmr3UdURERERETkZadZopZ5rkT0jz/+YMWKFVy4cIGYmBizbd99951VAhMREREREZH0Kcmr5i5fvpxKlSpx7Ngxvv/+e2JjYzly5AibNm3C3d09OWIUERERERFJE2wM1n2kV0lORMeMGcPkyZP58ccfsbe3Z+rUqRw/fpyWLVuSK1eu5IhRREREREQkTTAYDFZ9pFdJTkTPnDlDw4YNAbC3t+fBgwcYDAb69u3LvHnzrB6giIiIiIiIpC9JTkQzZcrEvXv3AMiePTuHDx8G4M6dO0RERFg3OhERERERkTTEYOVHepXkxYqqVq3Khg0bKFq0KK+//jq9e/dm06ZNbNiwgVq1aiVHjCIiIiIiImmCTTqeTmtNFieihw8fpkiRIsyYMYOoqCgAPvroI+zs7NixYwctWrRgyJAhyRaoiIiIiIiIpA8WJ6LFihWjbNmydO7cmVatWgFgY2PDBx98kGzBiYiIiIiIpCUqiFrG4mtEt2zZQuHChenfvz8+Pj60a9eO33//PTljExERERERkXTI4kS0SpUqfP7554SGhjJ9+nTOnTtHtWrVCAgIYNy4cYSFhSVnnCIiIiIiIi883b7FMkleNdfZ2ZkOHTqwZcsWTp48yeuvv87MmTPJlSsXTZo0SY4YRURERERE0gSDwbqP9CrJq+Y+Ll++fHz44Yf4+fkxePBg1q5d+9xjhYWFMXr0aNauXcvly5fx8vKiRIkS9OnTJ9VW4/X39+f8+fMAZMyYkcDAQAYPHszrr7+eKvFI6lswdyafz5tl1pbLLzdffbcGgEsXLzBzyqccDPmTmNgYKlSsTN/3P8QzcxZT//f79uD0iePcvn0LV1c3ypSvSLde/cia1StFz0Wez6H9+/hm2SJOnTjGrRvX+SR4MpWq1TRt37b5V9Z+/w2nThzjXvhdZi36mrwBBRKNc/TQARbNnc7xo4ewtbElT/5AxkyZjYODo6nP7u1bWbpwLmdPn8LewZ6iJcowbNyUlDhNSYIjB/bx/fIvOHPyGLdv3uCDkROpUKWGWZ+L5//ii7nTOHLgT+Lj48jpl4dBIyaQNZsPADHR0SycPYltm9YTGxNDiXIVebfPYDw8M5vGaFa9VKJj9/84mCq16ibvCYrVxcfHM2fWdNauWc3NGzfImtWLJs1epcs73U3Vj40b1vPNiuUcO3qEu3fvsHzlKgoUKJjKkUtyqV+nJqFXLidqb9nqLT4c8onpudFopGe3Lmzf9juTps6kZq3aKRmmiFU9dyK6detWPv/8c7799ltsbGxo2bIlnTp1eq6xzp07R1BQEB4eHkyYMIGiRYsSGxvLL7/8Qo8ePTh+/PhzjRsfH4/BYMDGJsmFX5MRI0bQpUsXwsPDmThxIm+88QbZs2enUqVKzz1mWhYTE4O9vX1qh5GqcufNx9RZ803PbW0f/jOKjIygb4+u5AsIZNqczwH4bPZ03u/bg3mLvjK9D0uVKUfbjl3JkiUr169dZcaUTxnyfl/mLlya8icjSRYVFUmefIHUbdSMEYP7Jd4eGUnh4iWpWqsuU8YOf+oYRw8d4KN+3WnVpiPd+32ArW0G/jp9AoPh799Vv//2K1PGDqfDu+9RonQ54uPjOffX6WQ7L3l+UVFR5M4bQO0GTRn78YBE20MvX+TD9zpRq0FT3uzwLk4Znbl47i/s7B1MfT6fOZE/dm1j4LBxZHR24bOp4xg7dABjZyw0G+u9QcMoVe7vvz/OLq7Jd2KSbBYu+Ixvvv6KEaPHkTdfPo4eOcwnQwbj4uLKW2+3BR7+TSlZqhR16tZnxDDdlSC9W7p8JQkJ8abnp0+d4t0uHXilTj2zfl8uWZy+S2TphG7fYpkkZWhXrlxhzJgxBAQEUL16dU6fPs20adO4cuUKn332GRUqVHiuILp3f/gN4J49e2jRogUBAQEULlyYfv36sWvXLlO/SZMmUbRoUZydncmZMyfdu3fn/v37pu2LFi3Cw8OD1atXU6hQIRwcHLhw4cJzxfSIq6sr3t7eBAQEMHPmTJycnPjxxx+Jj4+nU6dO5M6dGycnJwIDA5k6darZvps3b6ZcuXI4Ozvj4eFBUFCQqcJ64MABatSogaurK25ubpQuXZo//vjDtO+2bduoUqUKTk5O5MyZk169evHgwQPTdn9/f8aMGUPHjh1xdXUlV65czJs3z+z4O3bsoESJEjg6OlKmTBlWrVqFwWAgJCTE1Ofw4cPUr18fFxcXsmXLRps2bbhx44Zpe/Xq1enZsyd9+vQhS5Ys1K2rb95tbW3JnCWr6eGRKRMAB0P2ExZ6mSHDRpM3fwB58wcwZPgYjh89wr69u037t2rdjiJFi+Pt40vR4iV5u30njhw6QFxsbGqdkiRB2YqVaf9OT4KqPX2mRu36jXm747uULFv+H8eYO20CzV5/kzfadsI/Tz5y+vlTrVZd05c88XFxzJkyji49+9Lo1ZbkyOWPX+68VFPl64VUunwQrTv3oEKVmk/dvnT+TEqVD6L9u33Ik78APtlzUi6oGh6ZPAF4cP8ev/60io7d+1GsVDnyBRbivUHDOH74ACeOHDQby9nFlUyZs5ge9g4OTzukvOAOhOyneo1aVK1WnezZc/BKnXpUrFSZw4f+/nk3atKMd7r1pHzFiqkYqaQUT09PsmTJanps3fIbOXPmokzZcqY+x48fY8nizxk+ckwqRiqW0NRcy1iciNavXx8/Pz+mT5/Oq6++yrFjx9i2bRsdOnTA2dn5uQO4desW69ato0ePHk8dx8PD4+9gbWyYNm0aR44cYfHixWzatIn333/frH9ERATjxo1j/vz5HDlyBC8v6013zJAhA3Z2dsTExJCQkECOHDn45ptvOHr0KEOHDuXDDz9kxYoVAMTFxdGsWTOqVavGwYMH2blzJ127djVNuWndujU5cuRg79697Nu3jw8++AA7OzsAzpw5Q7169WjRogUHDx7k66+/Ztu2bfTs2dMsnokTJ1KmTBn2799P9+7d6datGydOnAAgPDycxo0bU7RoUf78809GjhzJoEGDzPa/c+cONWvWpGTJkvzxxx+sW7eOq1ev0rJlS7N+ixcvxt7enu3btzNnzhyrvZ5p1aULF2hStzqvN6nLsI/eJyz0CgCxsTEYDAbsHqsY2zs4YGNjw8GQP586VvjdO6z/eS1Fi5Ugw/9//pK+3bl1k+NHDuGRyZM+XdvyRsMaDOjekcMH/n6PnDp5jBvXr2Ew2NC9XUvebFyLj/p159yZU6kYuTyPhIQE/ti1Dd+cfgwb2J12zWoxsFtbdv3+m6nPmZPHiIuLo1jpv7+8yOGXm6zZvDlx1DwRnTd1LG2a1GTgu2349adVGI3GFDsXsZ7iJUqye/cuzp87C8CJ48fZ/+c+gqpUTeXI5EUQGxvDT2tW0/TVFqbPjZGRkXz4fn8GfzSULFmypnKEItZh8dRcOzs7Vq5cSaNGjbC1tbVaAKdPn8ZoNFKgQOJrqJ7Up08f0//7+/szatQo3n33XWbN+vuavdjYWGbNmkXx4sWtFiM8nJI6ceJE7t69S82aNbGzs2P48L+n3eXOnZudO3eyYsUKWrZsSXh4OHfv3qVRo0bkzZsXgIIF/76248KFCwwcONB03vnz5zdtCw4OpnXr1qbzzZ8/P9OmTaNatWrMnj0bR8eH15A1aNCA7t27AzBo0CAmT57Mb7/9RmBgIMuWLcNgMPDZZ5/h6OhIoUKFuHz5Ml26dDEdZ8aMGZQsWZIxY/7+Zu3zzz8nZ86cnDx5koCAANPxx48fb82XM80qVKQYHw0bTS5/f25ev87nn82me+e2LFnxA4WLFsfR0YlZ0ybybo8+GDEye/pk4uPjuXnjutk4s6ZN5NuvvyIqKpLCRYszYcqsfziipDePrgFasmAOXXr2I2/+QH5dt4YPenVl7pffkj2nH2GXLwHw5YI5dO01AG8fX1Z+9QUDe3ZmwdercXNzT81TkCS4e/sWUZERfLdsIa07dadt197s37ODcUMHMHLyPIqUKM3tWzfJYGeHi6v5NFuPTJm5feum6fmbHbtRrGRZHBwdCdm7i7mTxxIVGUmjFm+m9GnJf9Sxc1cePLhPs8b1sbW1JT4+np69+tKwkRZ9FNi08Vfu3btHk2avmto+HR9M8RIlqVFT14SmBel5pVtrsjgRXb16dbIEkJRvc3/99VeCg4M5fvw44eHhxMXFERUVRUREBBkzZgTA3t6eYsWK/es4hQsXNk2RrVKlCj///PM/9h00aBBDhgwhKioKFxcXxo4dS8OGDQGYOXMmn3/+ORcuXCAyMpKYmBhKlCgBPJxi0b59e+rWrcsrr7xC7dq1admyJT4+Dxem6NevH507d2bJkiXUrl2b119/3ZSwHjhwgIMHD7J06d/XDBqNRhISEjh79qwpoX38PA0GA97e3ly7dg2AEydOUKxYMVPSClCu3N/TOx4d57fffsPFxSXReZ85c8aUiJYuXfpfX8/o6Giio6PN22JtcUiHU8YqBlUx/X++/IEUKlqMFg1fYdOGdTRu1oKR4ybxafBIVi5fio2NDbXrNiCwQCGza/8A3mrTkUZNWxAWeoWF82YxcuhgJkydpV9cL4EEYwIADZq9Rt1GzQDIF1iQkD9288uaVXTs1puE//9efLNdZ6rUePiho/9HI3i7WR1+37Sehs20YFpa8ehvXLmg6jR5/W0A8uQP5PiRA/yyeiVFSvz779fHvdH27y8S8+QvQFRUJN8v/0KJaBq0ft3P/LTmR4LHTSRvvnycOH6MCeOCyerlRZOmrz57AEnXVn33LUGVq+LllQ2Azb9tZM/uXXy98vtUjkzEup5/FR8ryZ8/PwaD4ZkLEp07d45GjRpRrFgxvv32W/bt28fMmTOBh9XKR5ycnJ75Yf6nn34iJCSEkJAQ5s+f/699Bw4cSEhICJcuXeL27dum6a3Lly9nwIABdOrUifXr1xMSEkKHDh3MYlm4cCE7d+6kUqVKfP311wQEBJiueR02bBhHjhyhYcOGbNq0iUKFCvH99w9/wdy/f5933nnHFGNISAgHDhzg1KlTpmQVME3lfcRgMJCQkPCv5/O4+/fv07hxY7PjhISEcOrUKapW/Xt60LOmXgcHB+Pu7m72mDpxnMVxpGWurm7k9PPj0sWH1yKXrxjEN6vXsWbD76zduI2hI8dy/fpVfHPkMNvPI1Mmcvn5U65CJYYHf8rO7Vs5cuhAapyCpLDM/19B2c8/j1l7Tv/cXLv68H7Mj1ZZzpX77z729vZ4+2bnmu7ZnKa4untga5uBnH7mP+8cfrm5fu3hzzKTZ2biYmO5f++eWZ87t2+S6bFVc58UULAIN69fJfaxvzuSNkyeOJ4OnbtSr0FD8gcE0qhJM95u247P589N7dAklV25cpndu3bwaovXTG17du/i0sULVKlYltLFC1G6eCEABvR9j07t26RWqPIvbKz8SK/+0+1brMHT05O6desyc+ZMevXqlSjpuXPnDh4eHuzbt4+EhAQmTpxoWn300fWYSeXn52dx3yxZspAvX75E7du3b6dSpUqmqbHwsIr4pJIlS1KyZEkGDx5MxYoVWbZsmWlRp4CAAAICAujbty9vvvkmCxcu5NVXX6VUqVIcPXr0qce1VGBgIF9++SXR0dGmyuTevXvN+pQqVYpvv/0Wf39/MmR4/rfC4MGD6dfPfPXQe7HWm779IouIeMDlSxep18B8OtWjBYz27dnF7Vu3qFy1xtN2BzB9eRCjD5MvhWw+2cmcJSuXLpwza7984TxlKlYGIH+BQtjZ23PpwjmKFH94y464uFiuhl4hm7dPSocs/4GdnR35ChTi8sVzZu1XLl4w3bolb0BBMmTIwME/91Dp/4tgXb5wjutXwwgs9M8zfM6ePoGLq5vZdemSNkRFRSVaVdPGxpaEBF3z+7L74fvv8PTMTJWq1U1tHTt3pXkL85kwr73amAHvD6Za9X/+fCGpRzPcLJPqiSg8nOIaFBREuXLlGDFiBMWKFSMuLo4NGzYwe/Zsjh07Rr58+YiNjWX69Ok0btw41RfOyZ8/P1988QW//PILuXPnZsmSJezdu5fcuXMDcPbsWebNm0eTJk3w9fXlxIkTnDp1irZt2xIZGcnAgQN57bXXyJ07N5cuXWLv3r20aNECeDgduEKFCvTs2ZPOnTvj7OzM0aNH2bBhAzNmzLAovrfeeouPPvqIrl278sEHH3DhwgU+/fRT4O9/HD169OCzzz7jzTff5P3338fT05PTp0+zfPly5s+fb/G1wA4ODomm4cbcj7No37RmxuQJBFWtjrePLzeuX2P+3JnY2thSu14DANau/h6/3Hnw8MjEkUMHmPJpMG+81RY//4fviyOHDnLs6CGKlSiFm5s7ly9e4LM508meIydFipVIxTMTS0VGRHDl0t+rcYeFXubMyeO4urnj5e1DePhdroeFmq4Lvvj/hDNT5ix4Zs6CwWDgtdbtWTJ/NnnyBZInIJBff1rNxfPnGDJ6IgDOzi40bPY6S+bPJquXN17evqxctgiAKjXrpOj5yrNFRkQQevmi6fm1sMv8deoErm5uZM3mw6ut2vLp8A8oXLwURUuU4c89O9i7Yyujpjxc6dzZxZXaDZqxcNZEXN3ccMrozGfTxhNYuBiBhR8mont2bOHurVsEFCqKvb09Ift2s3Lp5zR7Q9WQtKhq9RrM/2wO3j6+D6fmHjvGl18spOmrLUx97t69Q2hoKNf/f8nN+bMPFzbKkiWLFqtJpxISEli96jsaN21mViB4tJLuk7x9fMmeI2dKhihiVS9EIponTx7+/PNPRo8eTf/+/QkNDSVr1qyULl2a2bNnA1C8eHEmTZrEuHHjGDx4MFWrViU4OJi2bdumSszvvPMO+/fv54033sBgMPDmm2/SvXt30/WmGTNm5Pjx4yxevJibN2/i4+NDjx49eOedd4iLi+PmzZu0bduWq1evkiVLFpo3b25a/KhYsWJs2bKFjz76iCpVqmA0GsmbNy9vvPGGxfG5ubnx448/0q1bN0qUKEHRokUZOnQob731lum6UV9fX7Zv386gQYOoU6cO0dHR+Pn5Ua9evf9079X07Nq1q3zy4UDC797BI5MnxUqUYu6iZWT6/20YLpw7y5wZkwm/excf3+y069iVN1q3M+3v6OjIlk2/smDuTKIiI8mcJSvlK1Zm5Nh3Xvr7s6YVJ48f4f2enU3P5057+AXPKw2aMGDISHb9vpmJo4eatgcPfTid/+2O79KmczcAmr/xNrHR0cyZNoF74XfJky+Q4Klz8H3sA0WXnn2xtbVl/IiPiImOJrBwUcZN/wxXN7cUOEtJitMnjvJx366m55/PnARAjbqN6T14OBWq1OTdfh/y7dKFzJ82Ad+cfgwaMYFCxUqa9unYoz8GGwPjhg4kNjaGkmUr8k6fwabtGWwz8NOqFSyYORGMRryz56Rj93680qh5yp2oWM0HHw5h5vSpBI8azq1bN8ma1YsWr7/BO916mPps/m0Tnwz5+z0waGBfAN7p1pNuPd5L8Zgl+e3auYPQ0Cs0e+wLCUmbbFQQtYjBaMFqQUlZqKhJE6349qJaunQpHTp04O7duzg5OSXrsW6k04qoPL/70XpPiLnImPhnd5KXhn+W578VnIi8HJzSyJ3u+q3+97VvkmpSk2ffXSQtsqgi2qxZM4sGMxgMxMfrg8WL4osvviBPnjxkz56dAwcOMGjQIFq2bJnsSaiIiIiIiMi/sSgRTcpKrPLiCAsLY+jQoYSFheHj48Prr7/O6NGjUzssEREREZF0S4sVWcaiqbkiSaWpufIkTc2VJ2lqrjxOU3NF5FnSytTcgWtOWHW8CY0CrTrei+K5Fit68OABW7Zs4cKFC4luOdGrVy+rBCYiIiIiIiLpU5IT0f3799OgQQMiIiJ48OABnp6e3Lhxg4wZM+Ll5aVEVEREREREXlqamWuZJN+jo2/fvjRu3Jjbt2/j5OTErl27OH/+PKVLlzbdp1JERERERETknyQ5EQ0JCaF///7Y2Nhga2tLdHQ0OXPmZPz48Xz44YfJEaOIiIiIiEiaYGMwWPWRXiU5EbWzs8PG5uFuXl5eXLhwAQB3d3cuXrxo3ehERERERETSEBsrP9KrJF8jWrJkSfbu3Uv+/PmpVq0aQ4cO5caNGyxZsoQiRYokR4wiIiIiIiKSjiQ5yR4zZgw+Pj4AjB49mkyZMtGtWzeuX7/OvHnzrB6giIiIiIhIWmEwWPeRXiW5IlqmTBnT/3t5ebFu3TqrBiQiIiIiIpJWpefrOq0pPU87FhERERERkRdQkiuiuXPnxvAvWf5ff/31nwISERERERFJq1QQtUySE9E+ffqYPY+NjWX//v2sW7eOgQMHWisuERERERGRNMdGiahFkpyI9u7d+6ntM2fO5I8//vjPAYmIiIiIiEj6ZrVrROvXr8+3335rreFERERERETSHBuDwaqP9MpqiejKlSvx9PS01nAiIiIiIiKSTiV5am7JkiXNFisyGo2EhYVx/fp1Zs2aZdXgRERERERE0pJ0XMS0qiQnok2bNjVLRG1sbMiaNSvVq1enQIECVg1OREREREQkLdFiRZZJciI6bNiwZAhDREREREREXhZJvkbU1taWa9euJWq/efMmtra2VglKREREREQkLTJY+b/0KskVUaPR+NT26Oho7O3t/3NAIiIiIiIiaZWm5lrG4kR02rRpABgMBubPn4+Li4tpW3x8PFu3btU1oiIiIiIiIvJMFieikydPBh5WROfMmWM2Ddfe3h5/f3/mzJlj/QhFRERERETSCFVELWPxNaJnz57l7NmzVKtWjQMHDpienz17lhMnTvDLL79Qvnz55IxVRERERERELDB27FgMBgN9+vQxtUVFRdGjRw8yZ86Mi4sLLVq04OrVq2b7XbhwgYYNG5IxY0a8vLwYOHAgcXFxZn02b95MqVKlcHBwIF++fCxatCjJ8SV5saLffvuNTJkyJflAIiIiIiIi6Z3BYLDq43ns3buXuXPnUqxYMbP2vn378uOPP/LNN9+wZcsWrly5QvPmzU3b4+PjadiwITExMezYsYPFixezaNEihg4daupz9uxZGjZsSI0aNQgJCaFPnz507tyZX375JUkxJjkRbdGiBePGjUvUPn78eF5//fWkDiciIiIiIpJu2Bis+0iq+/fv07p1az777DOzAuLdu3dZsGABkyZNombNmpQuXZqFCxeyY8cOdu3aBcD69es5evQoX375JSVKlKB+/fqMHDmSmTNnEhMTA8CcOXPInTs3EydOpGDBgvTs2ZPXXnvNdCmnxa9TUk9s69atNGjQIFF7/fr12bp1a1KHExERERERESvp0aMHDRs2pHbt2mbt+/btIzY21qy9QIEC5MqVi507dwKwc+dOihYtSrZs2Ux96tatS3h4OEeOHDH1eXLsunXrmsawVJJv33L//v2n3qbFzs6O8PDwpA4nIiIiIiKSbjznbNp/FB0dTXR0tFmbg4MDDg4OifouX76cP//8k7179ybaFhYWhr29PR4eHmbt2bJlIywszNTn8ST00fZH2/6tT3h4OJGRkTg5OVl0XkmuiBYtWpSvv/46Ufvy5cspVKhQUocTERERERFJN2wMBqs+goODcXd3N3sEBwcnOu7Fixfp3bs3S5cuxdHRMRXOPGmSXBH9+OOPad68OWfOnKFmzZoAbNy4ka+++opvvvnG6gGKiIiIiIi8rAYPHky/fv3M2p5WDd23bx/Xrl2jVKlSprb4+Hi2bt3KjBkz+OWXX4iJieHOnTtmVdGrV6/i7e0NgLe3N3v27DEb99Gquo/3eXKl3atXr+Lm5mZxNRSeIxFt3Lgxq1atYsyYMaxcuRInJyeKFSvGr7/+SrVq1ZI6nIiIiIiISLph7fuI/tM03CfVqlWLQ4cOmbV16NCBAgUKMGjQIHLmzImdnR0bN26kRYsWAJw4cYILFy5QsWJFACpWrMjo0aO5du0aXl5eAGzYsAE3NzfT7NeKFSvy008/mR1nw4YNpjEsleREFKBhw4Y0bNgwUfvhw4cpUqTI8wwpIiIiIiKS5ln7GlFLubq6JsrFnJ2dyZw5s6m9U6dO9OvXD09PT9zc3HjvvfeoWLEiFSpUAKBOnToUKlSINm3aMH78eMLCwhgyZAg9evQwJcPvvvsuM2bM4P3336djx45s2rSJFStWsHbt2iTFm+RrRJ9079495s2bR7ly5ShevPh/HU5ERERERESSweTJk2nUqBEtWrSgatWqeHt7891335m229rasmbNGmxtbalYsSJvv/02bdu2ZcSIEaY+uXPnZu3atWzYsIHixYszceJE5s+fT926dZMUi8FoNBqf5yS2bt3K/Pnz+e677/D19aV58+a0aNGCsmXLPs9wks7cuB+X2iHIC+Z+tN4TYi4yJj61Q5AXiH8W59QOQURecE52qR2BZWZuP2fV8XoE+Vt1vBdFkqbmhoWFsWjRIhYsWEB4eDgtW7YkOjqaVatWacVcMRMbn5DaIcgLJqvrs69tkJeLrbUvopE0reHspN1/TtK/td2Sdr2ZiKQtFk/Nbdy4MYGBgRw8eJApU6Zw5coVpk+fnpyxiYiIiIiIpCkGg3Uf6ZXFFdGff/6ZXr160a1bN/Lnz5+cMYmIiIiIiKRJmvBjGYsrotu2bePevXuULl2a8uXLM2PGDG7cuJGcsYmIiIiIiEg6ZHEiWqFCBT777DNCQ0N55513WL58Ob6+viQkJLBhwwbu3buXnHGKiIiIiIi88GwMBqs+0qsk377F2dmZjh07sm3bNg4dOkT//v0ZO3YsXl5eNGnSJDliFBERERERSRN0jahl/tN9RAMDAxk/fjyXLl3iq6++slZMIiIiIiIiko4l6fYt/8TW1pZmzZrRrFkzawwnIiIiIiKSJqXn6bTWZJVEVERERERERNL3dFpr+k9Tc0VERERERESSShVRERERERERK1GlzzJ6nURERERERCRFqSIqIiIiIiJiJQZdJGoRJaIiIiIiIiJWojTUMpqaKyIiIiIiIilKFVEREREREREr0X1ELaNEVERERERExEqUhlpGU3NFREREREQkRakiKiIiIiIiYiWamWsZVURFREREREQkRakiKiIiIiIiYiW6j6hllIiKiIiIiIhYiaacWkavk4iIiIiIiKQoVURFRERERESsRFNzLaNEVERERERExEqUhlpGU3NFREREREQkRakiKiIiIiIiYiWammsZJaIiIiIiIiJWoimnltHrJCIiIiIiIilKFVEREREREREr0dRcy6giKiIiIiIiIilKFVERERERERErUT3UMkpERURERERErEQzcy2jqbkiIiIiIiKSolQRTePat2/P4sWLE7WfOnWKfPny/aexN2/eTI0aNbh9+zYeHh7/aaz0Ij4+nkWfzWLDz2u5desGWbJkpV6jprTp+I7pwvSF82axacPPXL96lQx2GQgoUIjO3XpRqEgxs7F2btvKFwvmcOb0Sezt7SlesgyjP52WGqcl/9G+P/byxaIFHDt6hBvXrzNxygxq1Kpt2h4R8YBpkyeyedNG7t69g2/2HLzZug2vtWxl6nPx4gWmfDqe/fv3ERsTQ6WgKrw/eAiZs2RJjVMSK1qxfBkrvv6KK5cvA5A3X37e6dadylWqmfocCNnP9KmTOXToILY2NgQWKMjseQtwdHRMrbDlObUtl4N25XOatV24HUmHL0MA8HFz4N3K/hTxdcXO1sDe83eYseUctyNjTf3zZ3WmS6VcBGZzISHByNYzt5i97RxRsQmmPiVzuNGhQi5yZ85IVGw8649fZ8HOCyQYU+Q0xcoWfDaXjRvWc/bsXzg4OlKiREn69BuAf+48ANy9c4dZM6ezc8c2wkJDyZTJkxq1atPjvd64urqmcvTyJBtNzrWIEtF0oF69eixcuNCsLWvWrKkUTfr21Ref88O3Kxj8yWj88+TlxLEjjBv5Mc4urrR4ozUAOXP50Xvgh/hmz0F0VDTffLWEge+9w9Lv1uKRyROALZs28OmYYXTu1ptSZcoRHx/P2TOnUvPU5D+IiowkIKAATV9twYA+7yXaPnH8WPbu2c2osePx9c3Ozh3bGTt6BFmzelGtRk0iIyLo0bUT+QMLMHf+IgBmz5hGn/e6sXjp19jYaPJKWuaVzZvefQeQy88Po9HIjz+sonfPHnz97ffky5efAyH76f5OZzp2focPPvqYDLa2nDhxXD/3NOzszQgGrjpqeh7//+zQMYMN45sV4syNBwz4/uH2DhVyMqpxAXquOIQRyOxsx/hmhdh86gbTtpzF2d6W7lX8GVQ7H8N/PglAniwZGdOkIMv2XmbshtNkcbanT43c2BgMzN1+PsXPV/67P/bu4Y03W1O4aFHi4+KZPnUS73bpxHer15IxY0auXb/G9WvX6DdgEHnz5uPKlcuMGjGM69euMXGKvsR+0WhqrmX0Vy4dcHBwwNvb2+xha2vLli1bKFeuHA4ODvj4+PDBBx8QFxdn2i86OppevXrh5eWFo6MjlStXZu/evQCcO3eOGjVqAJApUyYMBgPt27dPjdN7oRw+GELlqjWoWLkqPr7ZqV6rDmXLV+LYkUOmPrXrNaRMuYr4Zs9J7rz56NFnIA8e3OfMqYcfIOLi4pg+aSzvvtefpi1aktPPH/88eanxSr3UOi35j4KqVKVHrz7UrPXKU7cfPBBC4ybNKFO2PL7Zc9Di9TfIHxDI4UMHAQgJ+ZMrVy4zfFQw+QMCyR8QyPDRYzl65DB7d+9KyVORZFC9Rk2qVK2Gn58//v65ea93XzJmzMjBAyEATBgXzJut29CpS1fy5cuPf+481K3XAHt7+9QNXJ5bfIKR2xGxpkd41MO/vYV9XMnm6sD4DWc4ezOCszcjGLfhNAFezpTM6Q5ABf9MxCckMG3zWS7dieLEtQdM2XyWqvky4+v+sEJeI39m/roRwZK9l7hyN4qDV8L5bPsFmhbzxslOH+3SotnzFtD01ebky5efwAIFGDF6LKGhVzh29AgA+fMHMGnqdKrXqEnOXLkoX6Ei7/Xuw5bNm8w+24mkJfptlU5dvnyZBg0aULZsWQ4cOMDs2bNZsGABo0aNMvV5//33+fbbb1m8eDF//vkn+fLlo27duty6dYucOXPy7bffAnDixAlCQ0OZOnVqap3OC6NIsRLs+2M3F8+fA+D0yRMcOvAn5StVfmr/2NhYfly1EmcXV/IGBAJw6sQxbly7ho2Ngc5vv07z+jV4v/e7/KWKaLpVrHgJtmzexLWrVzEajezds4sL589RoVIQADExMRgMBrPEw8HBARsbG/bv35daYUsyiI+P5+ef1hIZGUHx4iW5efMmhw4ewDNzZtq2bkWNqpXo2O5t/tz3R2qHKv9Bdg9Hvu5QmiVtSzK4Tj68XB7+27a3ffixKzb+7ym2MXEJGI1QxOfh9Eo7Wxti4408PsM2Ou5h/6JmfRJ4XHR8Ag4ZbAjwckmu05IUdP/ePQDc3N3/pc99XFxcyJBBExxfNAYr/5de6Z2bDqxZswYXl7//8NSvX5+AgABy5szJjBkzMBgMFChQgCtXrjBo0CCGDh1KZGQks2fPZtGiRdSvXx+Azz77jA0bNrBgwQIGDhyIp+fDaaReXl66RvT/3mrXiQcP7tO2ZRNsbGxJSIinc7devFKvkVm/Hb9vYcSQgURHRZE5S1YmzpiHh0cmAK5cvgTAos9m073PQLx9fFmxdDF93u3IlyvX/OsfHUmbBn34MaOGf0y92tXIkCEDBoOBj4eNpHSZsgAUK1YCJycnpk7+lJ69+oLRyLQpE4mPj+fG9eupHL1Yw6mTJ2jzVitiYqLJmDEjk6fNJG++fKaq6JyZM+g38H0CCxRkzQ+r6NqpPd/+sAY/P/9UjVuS7vjV+4z/9TSXbkfh6WxH23I5mdKiCJ2WhXA07B6RsfF0CfJjwc4LGIDOlXJha2Mgs/PDZHX/pbt0q+xHy5K+fHcgFEc7G7pUygWAp7MdAHvP36F5cR9q5M/MltM38cxoT5uyOR72yWiXKuct1pOQkMD4cWMoUbIU+fMHPLXP7du3mDdnFi1efyOFoxOxHiWi6UCNGjWYPXu26bmzszM9evSgYsWKpgV0AIKCgrh//z6XLl3izp07xMbGEhQUZNpuZ2dHuXLlOHbsWJKOHx0dTXR09BNtBhwcHJ7zjF5cv/36C7+uW8uQkePInScvp0+eYMakcWT+/6JFj5QsU5b5X67k7p3brF31LcMGD2D2wqVk8syMMeHht9hvd+hCtZoPp3IOGjqK1xvVZvPGX2jSvGWqnJskn+XLlnDo4AEmT5+Fj092/ty313SNaPmKlcjk6cm4iVMIHjmc5UuXYGNjQ936DSlQsJCuE0wn/P1zs+LbVdy/f48N63/h4w8HsWDRlyT8//fBay3foNmrLQAoWLAQu3fvZNV339K7b//UDFuew57zd0z//9dNOBZ2n2XtS1E9fxZ+PnqNET+fpE+NPLxa3BujETadvMHJa/dJMD6sgZ6/Fcm4X8/QrbIfnSvlIt5o5PsDYdx6EMP/u7Dv4l3mbT9Pnxp5GFwnPzHxCXy59xLFsruhtYrSvjGjhnPm1CkWLVn21O3379+nZ7d3yJM3L+9275nC0YkldI2oZZSIpgPOzs7/eYXc/yI4OJjhw4ebtfUbNIQBgz9OpYiSz5xpE3mrXSdq1XlYRc6TL4Cw0CssXTzfLBF1cspIjpy5yJEzF4WLFqd1i4b8tPp7WrfvTOYsDxeS8sud19Tf3t4e3+w5uBYWlrInJMkuKiqKGVOnMHHqdKpUrQ5AQGAgJ08c54vFn1O+YiUAKlaqzOqfN3D79m0y2Nri6ubGK9Urkz1Hzn8ZXdIKO3t7cvn5AVCocBGOHD7E0i+/oGPnLgDkyZvXrH/uPHkJC72S4nGK9T2IiefSnSjT9Z37Lt6lzRf7cXPMQHyCkQcx8XzTsTShd//+QnfTyRtsOnmDTE52RMbFgxFeK+HDlfAoU5+VIaGsDAkls7Md96Li8XZzoEslP0LvRiWKQdKOMaNGsHXLZj5f/CXZvL0TbX/w4D7d3+mMs7Mzk6fNxM5OFfAXkVbNtYy+ak+nChYsyM6dOzEa//5udPv27bi6upIjRw7y5s2Lvb0927dvN22PjY1l7969FCpUCMB0vVp8fPy/Hmvw4MHcvXvX7PFev/eT4axSX3RUFDYG8382tra2GJ+xXr4xIYGYmBgAAgoUws7e3nSdKUBcXCxhoZfJ5uNj9ZgldcXFxREXF5vofWNjY2Oqjj8uU6ZMuLq5sWf3Lm7dukm16jVSKlRJQQkJCcTGxJA9ew6yenlx7uxZs+3nz53Dxzd7KkUn1uRoZ4OvuyO3HsSYtYdHxfEgJp4SOdzwyGjHjrO3Eu17OzKWqNgEqufPTEx8Avsu3E3U5+aDWGLiE6gZkIWr96I5df1Bsp2LJB+j0ciYUSPYtHEDn32+mBxP+RLy/v37vNulE3Z2dkydMTtdzjyTl4sqoulU9+7dmTJlCu+99x49e/bkxIkTfPLJJ/Tr1w8bGxucnZ3p1q2b6VrQXLlyMX78eCIiIujUqRMAfn5+GAwG1qxZQ4MGDXBycjK7FvURBweHRL8MHxhjEvVLDypWqcaSRfPw8vbBP09eTp84zoplX9CgcTMAIiMj+HLhZ1SqUp3MWbJy985tVq1czvXr16heqw4Azi4uNGnekoWfzcQrmzfZfHxYvmQRgKmPpC0REQ+4eOGC6fnly5c4cfwYbu7u+Pj4UrpMWaZMmoCDowM+PtnZ98ce1v74A/0GfmDa54fvvyV3nrxk8vTkYEgIn44bTes27Uz3kJO0a+rkiVSuUhVvHx8iHjzgp7Vr+GPvHmbPW/BwRfIOnZg9czqBgQUILFCQ1T98z7mzfzFxsm7JkBa9E+THzrO3uXovmszOdrQvn5MEo5FNJ28AULdgVi7ciuROZCyFfVzpUcWfb0NCuXTn70pm02LeHA19eD1p6ZzudA3yY/6OCzyI+fuL4ZYlfdl74Q4JRiNV8nrSqrQvI9ed1H1E06gxI4fz809rmDJ9Fs4ZnU3rA7i4uuLo6Pj/JLQjUVGRjBk7gQf37/Pg/n0AMnl6Ymtrm5rhyxM0NdcyBuPjJTNJc9q3b8+dO3dYtWpVom1btmxh4MCBHDhwAE9PT9q1a8eoUaNMq6tFRUXx/vvv89VXX3Hv3j3KlCnD5MmTKVu2rGmMkSNHMmvWLK5evUrbtm1ZtGiRRXGF3k2fiWjEgwcsmDuDbZs3cvv2LbJkyUrNOvVp17kbdnZ2REdHM+rjQRw7coi7d27j5u5BgUKFadPxHQoUKmIaJy4ulnkzp7Lh5x+Jjo6mYOGi9Ow7iNx5U2+KdXJzc0q/04f+2Lubrh3bJWpv3KQZw0eP5caN60yfMoldO7cTfvcuPj6+NH+tJa3btjddxz1t8kR+/OF77t69i292X157vZXZ9vTI1ib9ntvjPvn4Q/bs2sX169dwcXUlICCQDp26ULHS39foL/hsHl8vX8rdu3cJDCxAn34DKFW6TCpGnfIazt6Z2iFYxZC6+Snq64abUwbuRsZy+Mo9Fuy8QGj4w6m3nSvlom6BrLg6ZuBqeDQ/Hr7KypBQszEGvZKPCn4eONrbcvF2JCv+vMKvJ26Y9fm0WSHyezljZ2vDmRsPWLLnktn1qenB2m4VUzuEFFO8cOBT20eMCqbpq83Zu2c3nTu0fWqfn9ZvJHv2HMkZ3gvDMY2U0NYfs+5Cg3UKZrXqeC8KJaKSLNJrIirPLz0novJ8XpZEVCyTXhJRsZ6XKREVyygRTV/SyI9TRERERETkxZee7/1pTUpERURERERErEQTfiyjVXNFREREREQkRakiKiIiIiIiYiWammsZVURFREREREQkRakiKiIiIiIiYiXp+M5rVqVEVERERERExEo0NdcymporIiIiIiIiKUoVURERERERESvR7Vsso0RURERERETESjQ11zKamisiIiIiIiIpShVRERERERERK9GquZZRIioiIiIiImIlykMto6m5IiIiIiIikqJUERUREREREbESG83NtYgqoiIiIiIiIpKiVBEVERERERGxEtVDLaNEVERERERExFqUiVpEU3NFREREREQkRakiKiIiIiIiYiUGlUQtokRURERERETESrRormU0NVdERERERERSlCqiIiIiIiIiVqKCqGVUERUREREREZEUpYqoiIiIiIiItagkahEloiIiIiIiIlaiVXMto6m5IiIiIiIikqJUERUREREREbES3b7FMqqIioiIiIiIWInByg9LBQcHU7ZsWVxdXfHy8qJZs2acOHHCrE9UVBQ9evQgc+bMuLi40KJFC65evWrW58KFCzRs2JCMGTPi5eXFwIEDiYuLM+uzefNmSpUqhYODA/ny5WPRokVJiPQhJaIiIiIiIiJp3JYtW+jRowe7du1iw4YNxMbGUqdOHR48eGDq07dvX3788Ue++eYbtmzZwpUrV2jevLlpe3x8PA0bNiQmJoYdO3awePFiFi1axNChQ019zp49S8OGDalRowYhISH06dOHzp0788svvyQpXoPRaDT+99MWMRd6Nya1Q5AXjJuTXWqHIC8YWxvNXZK/NZy9M7VDkBfM2m4VUzsEecE4ppGLCv88H27V8Ur5uT3XftevX8fLy4stW7ZQtWpV7t69S9asWVm2bBmvvfYaAMePH6dgwYLs3LmTChUq8PPPP9OoUSOuXLlCtmzZAJgzZw6DBg3i+vXr2NvbM2jQINauXcvhw4dNx2rVqhV37txh3bp1FseniqiIiIiIiIiVGKz83/O6e/cuAJ6engDs27eP2NhYateubepToEABcuXKxc6dD78M3LlzJ0WLFjUloQB169YlPDycI0eOmPo8PsajPo/GsFQa+V5BRERERETk5RMdHU10dLRZm4ODAw4ODv+4T0JCAn369CEoKIgiRYoAEBYWhr29PR4eHmZ9s2XLRlhYmKnP40noo+2Ptv1bn/DwcCIjI3FycrLovFQRFRERERERsRKDwbqP4OBg3N3dzR7BwcH/GkOPHj04fPgwy5cvT6GzTjpVREVERERERF5QgwcPpl+/fmZt/1YN7dmzJ2vWrGHr1q3kyJHD1O7t7U1MTAx37twxq4pevXoVb29vU589e/aYjfdoVd3H+zy50u7Vq1dxc3OzuBoKqoiKiIiIiIhYjbVv3+Lg4ICbm5vZ42mJqNFopGfPnnz//fds2rSJ3Llzm20vXbo0dnZ2bNy40dR24sQJLly4QMWKDxcHq1ixIocOHeLatWumPhs2bMDNzY1ChQqZ+jw+xqM+j8aw+HXSqrmSHCJjUzsCedHoV408yaA7fstj9HaQJx26eDe1Q5AXTNnc7qkdgkUOXLxn1fGK53S1qF/37t1ZtmwZP/zwA4GBgaZ2d3d3U6WyW7du/PTTTyxatAg3Nzfee+89AHbs2AE8vH1LiRIl8PX1Zfz48YSFhdGmTRs6d+7MmDFjgIe3bylSpAg9evSgY8eObNq0iV69erF27Vrq1q1r8XkpEZVkoURUnqRfNfIkJaLyOL0d5ElKROVJSkT/3T/9XV24cCHt27cHICoqiv79+/PVV18RHR1N3bp1mTVrlmnaLcD58+fp1q0bmzdvxtnZmXbt2jF27FgyZPj7qs7NmzfTt29fjh49So4cOfj4449Nx7CUElFJFkpE5Un6VSNPUiIqj9PbQZ6kRFSelFYS0YMX71t1vGI5Xaw63otCixWJiIiIiIhYib5Ys4wWKxIREREREZEUpYqoiIiIiIiIlaggahkloiIiIiIiItaiTNQimporIiIiIiIiKUoVURERERERESsxqCRqEVVERUREREREJEWpIioiIiIiImIlun2LZZSIioiIiIiIWInyUMtoaq6IiIiIiIikKFVERURERERErEUlUYsoERUREREREbESrZprGU3NFRERERERkRSliqiIiIiIiIiVaNVcy6giKiIiIiIiIilKFVERERERERErUUHUMkpERURERERErEWZqEU0NVdERERERERSlCqiIiIiIiIiVqLbt1hGiaiIiIiIiIiVaNVcy2hqroiIiIiIiKQoVURFRERERESsRAVRyygRFRERERERsRZlohbR1FwRERERERFJUaqIioiIiIiIWIlWzbWMKqIiIiIiIiKSolQRFRERERERsRLdvsUySkRFRERERESsRHmoZTQ1V0RERERERFKUKqIiIiIiIiLWopKoRdJ8RdTf358pU6b8ax+DwcCqVasAOHfuHAaDgZCQkGSPTUREREREXi4GK/+XXqVqRbR9+/YsXrwYADs7O3LlykXbtm358MMPyZAheULLmTMnoaGhZMmSJcn7Dhs2jOHDh/9rH6PR+LyhvZAWLVpEnz59uHPnTmqH8sKaPXM6c2fPMGvzz52bVT+uA2DlN1/z89o1HD92hAcPHrB1x17c3NxSI1RJJvv+2MsXixZw9OgRbly/zqQpM6hRq7Zp+9CPPuDH1avM9qkUVJmZc+YDcOXyJebNnc3ePbu4eeMGWbN60aBRYzp3fZf/tXfncVFV/R/APwMywzIs7uxgIoo94m4hIWIYaC64hJophEuCCC6k0JOKW4iCuTyo+dQDaJpoFLkkZiqWuC9gGSCaiinmjiKIOHN+f/Dj5gDqqAgun/frNa/X3HvPPfd7l7lzzz3nnqunJ6/JVaFnoMc73ZB/4Xyl8b6D38cnn07HlSuX8XnMPOzbuwe3i27D3r4JRo4eA8/uXrUQLdUGlUqFZXFLsHnThrJzQKNG6NO3H0aPCYKMvZ688DasTcDB9J3I/+ss5HIFmrVshUEB42BpYwcAKLxVgORVK/Db4f24evlvmJiaob2LOwb6jYGhkVLK58qli4hfEo2sY4egr2+ItzzfxaCAIOjqVr5mPnE8E7M/HgNr+9fw2dLVNbauRI+j1pvment7Iz4+HiUlJfjxxx8xduxY6OnpISIi4pksT1dXF+bm5k80b1hYGMaMGSMNd+zYEaNHj8aoUaOqTH/37l3I5S/uRWRpaWlth/DCaOrQDF98GS8N6+rqSt/v3CmG61tucH3LDYsXxtZGePSMFRcXw9GxBfr2G4BJ48dVmaazqxtmzP5MGpbfV8A8ffo0hFqNT6fNgI2NHU6ezMWsyKkoLi7GxLApzzx+erZWr/0WarVKGj6Zm4sxoz5E93e8AQCfRkzBrVs3sfA/y1DXrC62/LgRkyeNx5qkZLRwallbYVMNiv/qv1if9A1mfRaNpg4O+OP33zHt0wgojY0x9IPhtR0ePaWs346ge+/38JqjE1RqFdbFL0P0v8chekUS9PUNcP3qFdy4egXvjwqFlW0TXLmUj/glc3H92hWEfjoXAKBWqRAzbQLM6tbH9AVf4ca1K1geEwndOnUw6MMgjeXdLryF5TGReL1NBxTcuFYbq/zK4/0j7dR601yFQgFzc3PY2dkhMDAQnp6e2LBhAwCga9euGD9+vEZ6Hx8f+Pv7a4y7desWhgwZAiMjI1hZWSEuLu6By6uqae7x48fRq1cvmJiYwNjYGG5ubjh16lSleZVKJczNzaWPrq4ujI2NpeHBgwcjODgY48ePR4MGDeDlVXY3e8GCBWjVqhWMjIxgY2ODoKAgFBYWSvkmJCTAzMwMW7duhZOTE5RKJby9vZGfny+lSUtLQ6dOnWBkZAQzMzO4urri7NmzAMpqatu0aYMvvvgCNjY2MDQ0hK+vLwoKCqT51Wo1Zs6cCWtraygUCrRp0wapqamVtktSUhLc3d2hr6+P1atX48MPP0RBQQFkMhlkMhkiIyMfuG1fZbq6umjQoKH0qVu3njTtg2H+CBg5Gq2cW9dihPQsveXWBWNDxqPb290fmEYul2scIyamptI017fcMGN2FFw6vwVrGxt09eiG4f4B2PHztpoIn56xevXqaez7X3bthI2NLTp07AQAyMw4iiHvf4BWrZxhbWODUR8FwdjYBH8cP17LkVNNycg4iq7d3kYX966wsrJGdy9vuHR+C7//dqy2Q6NqMGXOYnR5pxes7ZvC7jVHfDRpGq5euogzuVkAABv7pgidGo12b7qhsaU1Xm/TEe/5BeLo/l+hUt0DAPx2ZD/O551G4OQZsGvqiNYdO2Pg8I/w88b1uFeh4iB+yVy4dPWCg1OrGl9XKiOr5s/LqtYLohUZGBjg7t27jzXP/Pnz0bp1axw9ehTh4eEIDQ3Ftm3aXcCdP38eXbp0gUKhwI4dO3D48GEEBATg3r17TxI+EhMTIZfLkZ6ejuXLlwMAdHR0sHjxYhw/fhyJiYnYsWMHJk+erDFfUVERYmJisGrVKvzyyy/Iy8tDWFgYAODevXvw8fGBu7s7jh07hr1792L06NEazXVOnjyJdevWYePGjUhNTcXRo0cRFPTPHbJFixYhNjYWMTExOHbsGLy8vNCnTx/k5uZqxFG+/bKysuDh4YGFCxfCxMQE+fn5yM/Pl2IiTXl5Z9Hd4y286/02IqZMQn7+hdoOiZ4zhw4dQDf3zvDp7Y05syJx48b1h6YvvHVLo7BKL4fS0rv4cdMG9O03QDqHt27TFltTt6Cg4AbUajVSf9yMkrsl6NCpUy1HSzWlTZu2OLBvH86cOQ0AyMnOxtGjh/GWW5dajoyehaKissoII+MHn+OLbhfCwNBIanabm/UbbOybwrRufSlNq/ZvorjoNv46+6c0btdPG3Ep/zz6fzDyGUVPVH1qvWluOSEEtm/fjq1bt2LcuKqbtj2Iq6srwsPDAQCOjo5IT0/H559/ju7dH1w7US4uLg6mpqZYu3Yt9PT0pDyeVLNmzTBv3jyNcffX6trb22P27NkYM2YMli5dKo0vLS3F8uXL0bRpUwBAcHAwZs6cCQC4efMmCgoK0KtXL2m6k5OTxjLu3LmDlStXwsrKCgCwZMkSvPvuu4iNjYW5uTliYmIwZcoUDB48GAAQHR2NnTt3YuHChRo1yOPHj0f//v2lYVNTU8hksiduzvwqaOXsjJmzo2Bv3wRXrlzG8qVxCBg+FN+mbITRfc920Kur81tu6Ob5DqysrPDXuXNYsvhzBAeORuLXazWacZfLyzuLtd98jQmTJleRG73Idmz/Gbdu3UIfn37SuHmxCzElbALcXd9AnTp1oK+vjwUL/wNbW7tajJRqUsDI0SgsLIRPrx7Q1dWFSqXCuNAJeLdXn9oOjaqZWq3G18sXwLFla9jYN60yza2CG0j55n/w6OEjjSu4fhWmZvU00pma1ZemAcDF83lI+l8cpsZ8UeVzo1Rz2DRXO7V+lG7atAlKpRKlpaVQq9V4//33H7v5p4uLS6XhR/WkWy4jIwNubm5SIfRptW/fvtK4n3/+GVFRUcjOzsbNmzdx79493LlzB0VFRTA0NAQAGBoaSoVMALCwsMClS5cAlDXr8vf3h5eXF7p37w5PT0/4+vrCwsJCSm9raysVQoGybaBWq5GTkwNDQ0NcuHABrq6uGnG5uroiMzNTY1yHDh0ee51LSkpQUlKiMU6to4BCoXjsvF5Eb7m5S98dm7fAv1q1Rs93PPBT6hb0G/BeLUZGzwvvHu9K35s5Nkczx+bo3bM7Dh08gDfe1Dx/Xfr7bwSPGQXPd7zRf6BvTYdKz1jKd8lwfasLGjVqLI1b+p9FuHXrJr74MgFmZnWxc8fPmBw2HvGJq9HMsXktRks1ZWvqFvy4eSOi5sXCwcEB2dlZmD83Cg0bNtK4aUEvvsS4efjrzJ+YGruiyulFtwsRM20CrGyboP8Ho7XOV61SIW7uVAwYNgoW1ryJRS+GWm+a6+HhgYyMDOTm5qK4uBiJiYkwMjICUNaktWIvtNXdgY6BgUG15lcee7kzZ86gV69ecHZ2RnJyMg4fPizVQN7fBLliQVgmk2mse3x8PPbu3YvOnTsjKSkJjo6O2LdvX7XGXlX82oiKioKpqanGZ350VLXH9qIwMTGBrZ09zuXl1XYo9JyytrGBWd26OJd3VmP8pUt/Y9SI4XBu0xZTp8+spejoWblw4Tz279uDfgMGSuPO5eVh7ZqvETnrM7zxpguat2iBMUHBeP31fyHpG/Z0+ar4PHYeAkaMRo+e75bdqOrjgw+G++GrL7+o7dCoGiXGzcfR/bvxybylqN+wcaXpxUW3Mf/TUOgbGGL8tHkab5AwrVu/UsdDBTeuStOKi4twOjcLiXExGN7TBcN7uiBlzVfI+zMXw3u64HjGwWe7clQBnxLVRq3XiBoZGcHBwaHKaQ0bNtTosEelUuH333+Hh4eHRrqKBbJ9+/ZVarr6IM7OzkhMTERpaWm11Yre7/Dhw1Cr1YiNjYWOTlm5f926dU+UV9u2bdG2bVtERETAxcUFa9aswZtvvgkAyMvLw4ULF2BpaQmgbBvo6OigefPmMDExgaWlJdLT0+Hu/k/tXXp6Ojo94hkkuVwOlUr10DQRERGYOHGixji1zqtRG1qVoqLb+OvcOTTo3bC2Q6Hn1N8XL6Lgxg00aNhIGnfp77JCqFPL1zFj1mfS+YJeHj98/x3q1asPty5dpXF37hQDAHRkmvtbR0cX6pfsdWD0YHeK70BHR/NiU1dXF2o1j4GXgRACK5fG4NCeNPx73jI0MreqlKbodiHm/TsEdfTkmBgZC7lc8zqqmVMr/LA2HgU3rklNdH8/cgAGhkawsm0C3Tp1ELX8G415ft70Lf7IOISQT+eiobnls1tBqoRNc7VT6wXRh+nWrRsmTpyIzZs3o2nTpliwYEGV77NMT0/HvHnz4OPjg23btmH9+vXYvHmzVssIDg7GkiVLMHjwYERERMDU1BT79u1Dp06d0Lz50zeJcnBwQGlpKZYsWYLevXtrdGKkrdOnT2PFihXo06cPLC0tkZOTg9zcXAwf/k+X7vr6+vDz80NMTAxu3ryJkJAQ+Pr6Ss92fvzxx5g+fTqaNm2KNm3aID4+HhkZGVi9+uF33O3t7VFYWIjt27ejdevWMDQ0lJoTl1MoKjfDLX6F3vyyYH40unT1gIWlJS5fuoRlcUugq6sD7569AABXrlzGlStXpBrSk7knYGhkBAsLC5iamtVi5FRdiopua9SAnz//F3Kys2Dy/y0EvlgWh7c930GDBg1w7tw5LFowHza2tujs+haAskLoyIDhsLCwxMRJU3D9+j93vRs04A2Nl4FarcaGlO/Qu6+PRi2HfZPXYGNrh9kzp2FC2BSYmZph546fsW9vOhbHsTbsVeHe1QP/XbEc5haWaOrggOysLKxKjEfffgNqOzSqBglx87B351ZMmB4DfQND3Lh2BQBgaKSEXKGPotuFiP53CO7euYPAyTNRXFSI4v/v0MjEtC50dHXRqt0bsLJtguXzpmPwyHEouHYV3yYuh2fv96D3/68KrPjMqYlpXejJ5Q98FpWotj3XBdGAgABkZmZi+PDhqFOnDiZMmFCpNhQAJk2ahEOHDmHGjBkwMTHBggULpFenPEr9+vWxY8cOfPzxx3B3d4euri7atGlT6XnKJ9W6dWssWLAA0dHRiIiIQJcuXRAVFaVRiHwUQ0NDZGdnIzExEVevXoWFhQXGjh2Ljz76SErj4OCA/v37o2fPnrh27Rp69eql0RlSSEgICgoKMGnSJFy6dAktW7bEhg0b0KxZs4cuu3PnzhgzZgwGDRqEq1evYvr06XyFSwV//30REZMn4saNG6hbrx7atm2PlavXoV69sjuW65PW4otl/5HSB/gNBQDMmB2Fvj79q8yTXix/HP8dowL8pOHY+WXvfevdxwefTI1E7okcbNyQgls3b6Fho4ZwcXFFUHCo9J7hfXvTcS7vLM7lnYWXp7tG3kd/y665FaFnZt/ePcjPvwCfCgULPT09/GfZCiz+PBahY8egqLgItja2mDVnLty6uD8gN3rZhP/7U8QtXoTPZs3AtWtX0bBRIwx8bxA+Chxb26FRNdi+KRkAMGfyGI3xoydOQ5d3euHMyRycyv4dADApQPO64POEFDQ0t4SOri7CZixA/H+iMWPCCCj0DeDm+S4GDtf+OVKqOawQ1Y5MVHwIk144kZGRSElJ0Xg3am17lWpESTs81VBFMrZdovvwcKCKfjtX8OhE9Erp2OTFeK1ZfsHjvYryUSxM5dWa3/OCDyERERERERFRjXqum+YSERERERG9SGRsnKsVNs2lZ4JNc6kinmqoIjbNpfvxcKCK2DSXKnpRmuZeLKjeC2Fz0+p/s8fzgDWiRERERERE1YU31rTCgigREREREVE1YTlUO+ysiIiIiIiIiGoUa0SJiIiIiIiqCZ951w4LokRERERERNWEveZqh01ziYiIiIiIqEaxRpSIiIiIiKi6sEJUKyyIEhERERERVROWQ7XDprlERERERERUo1gjSkREREREVE3Ya652WCNKRERERERENYo1okRERERERNWEr2/RDguiRERERERE1YRNc7XDprlERERERERUo1gQJSIiIiIiohrFprlERERERETVhE1ztcMaUSIiIiIiIqpRrBElIiIiIiKqJuw1VzusESUiIiIiIqIaxRpRIiIiIiKiasJnRLXDgigREREREVE1YTlUO2yaS0RERERERDWKNaJERERERETVhVWiWmFBlIiIiIiIqJqw11ztsGkuERERERER1SjWiBIREREREVUT9pqrHRZEiYiIiIiIqgnLodph01wiIiIiIiKqUawRJSIiIiIiqi6sEtUKa0SJiIiIiIioRrFGlIiIiIiIqJrw9S3aYUGUiIiIiIiomrDXXO2waS4RERERERHVKJkQQtR2EEQvq5KSEkRFRSEiIgIKhaK2w6FaxuOBKuIxQRXxmKCKeEzQy4oFUaJn6ObNmzA1NUVBQQFMTExqOxyqZTweqCIeE1QRjwmqiMcEvazYNJeIiIiIiIhqFAuiREREREREVKNYECUiIiIiIqIaxYIo0TOkUCgwffp0di5AAHg8UGU8JqgiHhNUEY8JelmxsyIiIiIiIiKqUawRJSIiIiIiohrFgigRERERERHVKBZEiR7hzJkzkMlkyMjIqJHlde3aFePHj6+RZdGLJTIyEm3atKntMIiIXlr29vZYuHDhQ9PIZDKkpKQAqPlrBKKXCQui9Mrz9/eHTCaTPvXr14e3tzeOHTsGALCxsUF+fj7+9a9/1XKk9DguXryIcePG4bXXXoNCoYCNjQ169+6N7du311pM9vb20nFmYGAAe3t7+Pr6YseOHVrNHxYWVqvxPy+e931rZGSEdu3aYf369bUWDz0fKv6/lH9Onjz51HmnpaVBJpPhxo0bTx/oS+L+7S2Xy+Hg4ICZM2fi3r17z2yZT3ONEBkZWeXxcf/nZZOQkAAzM7PaDoOeEyyIEgHw9vZGfn4+8vPzsX37dtSpUwe9evUCAOjq6sLc3Bx16tSp5ShJW2fOnEH79u2xY8cOzJ8/H7/99htSU1Ph4eGBsWPHPnG+KpUKarX6qWKbOXMm8vPzkZOTg5UrV8LMzAyenp6YM2fOA+cRQuDevXtQKpWoX7/+Uy3/Ue7evftM839aL8K+PXr0KDp27IhBgwZhz549T5Xni+x5P5Zqyv3/L+WfJk2a1HZYL63y7Z2bm4tJkyYhMjIS8+fPf2bLe5prhLCwMI3jwtraWjqPlH/u96L/pkpLS2s7BHrOsCBKhLKu0c3NzWFubo42bdogPDwc586dw+XLl6tsdnP8+HH06tULJiYmMDY2hpubG06dOoVffvkFenp6uHjxokb+48ePh5ubmzScnp6Orl27wtDQEHXr1oWXlxeuX79eZWwlJSUICwuDlZUVjIyM8MYbbyAtLe1ZbIaXRlBQEGQyGQ4cOIABAwbA0dERr7/+OiZOnIh9+/ZJ6RYsWIBWrVrByMgINjY2CAoKQmFhoTS9/M7thg0b0LJlSygUCuTl5T1VbMbGxjA3N4etrS26dOmCFStWYOrUqZg2bRpycnIA/FPTsWXLFrRv3x4KhQK7d+/WaJr7008/QV9fv1JtSGhoKLp16yYN7969G25ubjAwMICNjQ1CQkJw+/Ztabq9vT1mzZqF4cOHw8TEBKNHj36q9XvWXoR96+joiLi4OBgYGGDjxo1QqVQYMWIEmjRpAgMDAzRv3hyLFi3SmDctLQ2dOnWCkZERzMzM4OrqirNnzwIAMjMz4eHhAWNjY5iYmKB9+/Y4dOiQNK82+/izzz5DQEAAjI2NYWtrixUrVmgsf8+ePWjTpg309fXRoUMHpKSkVDrv/f777+jRoweUSiUaN26MYcOG4cqVK9L0rl27Ijg4GOPHj0eDBg3g5eX1VNvzZXH//0v5R1dXF7t27UKnTp2gUChgYWGB8PBwjZq7kpIShISEoFGjRtDX18dbb72FgwcPAii7IePh4QEAqFu3LmQyGfz9/Wtj9Z475dvbzs4OgYGB8PT0xIYNGwBU/eiLj49PpW1369YtDBkyBEZGRrCyskJcXNwDl/c41wgVKZXKSsdF+XnE3NwcgwcPrvI3pe35bevWrXBycoJSqZQK6OUeds4p/6/54osvYGNjA0NDQ/j6+qKgoECaX61WY+bMmbC2toZCoUCbNm2QmppaabskJSXB3d0d+vr6WL16NT788EMUFBRINb6RkZEP3Lb08mNBlKiCwsJCfP3113BwcKiy9un8+fPo0qULFAoFduzYgcOHDyMgIAD37t1Dly5d8Nprr2HVqlVS+tLSUqxevRoBAQEAgIyMDLz99tto2bIl9u7di927d6N3795QqVRVxhMcHIy9e/di7dq1OHbsGN577z14e3sjNzf32WyAF9y1a9eQmpqKsWPHwsjIqNL0+5sE6ejoYPHixTh+/DgSExOxY8cOTJ48WSN9UVERoqOj8eWXX+L48eNo1KhRtcccGhoKIQR++OEHjfHh4eGYO3cusrKy4OzsrDHt7bffhpmZGZKTk6VxKpUKSUlJGDp0KADg1KlT8Pb2xoABA3Ds2DEkJSVh9+7dCA4O1sgrJiYGrVu3xtGjRzF16tRqX7/q8iLt2zp16kBPTw93796FWq2GtbU11q9fjz/++APTpk3DJ598gnXr1gEA7t27Bx8fH7i7u+PYsWPYu3cvRo8eLTXLGzp0KKytrXHw4EEcPnwY4eHh0NPTA6D9Po6NjUWHDh1w9OhRBAUFITAwULrxcfPmTfTu3RutWrXCkSNHMGvWLEyZMkVj/hs3bqBbt25o27YtDh06hNTUVPz999/w9fXVSJeYmAi5XI709HQsX7682rbny+b8+fPo2bMnOnbsiMzMTCxbtgxfffUVZs+eLaWZPHkykpOTkZiYiCNHjsDBwQFeXl64du0abGxspN9+Tk4O8vPzK93coDIGBgaPXZM4f/586ZwYHh6O0NBQbNu2Tat5H3aN8CSq+k1pe36LiYnBqlWr8MsvvyAvLw9hYWEAHn3OAYCTJ09i3bp12LhxI1JTU6VzR7lFixYhNjYWMTExOHbsGLy8vNCnT59K1ybl2y8rKwseHh5YuHAhTExMpBrf8pjoFSWIXnF+fn5CV1dXGBkZCSMjIwFAWFhYiMOHDwshhDh9+rQAII4ePSqEECIiIkI0adJE3L17t8r8oqOjhZOTkzScnJwslEqlKCwsFEIIMWTIEOHq6vrAeNzd3UVoaKgQQoizZ88KXV1dcf78eY00b7/9toiIiHjSVX6p7d+/XwAQ33333WPPu379elG/fn1pOD4+XgAQGRkZ1RKbnZ2d+Pzzz6uc1rhxYxEYGCiEEGLnzp0CgEhJSdFIM336dNG6dWtpODQ0VHTr1k0a3rp1q1AoFOL69etCCCFGjBghRo8erZHHr7/+KnR0dERxcbEUk4+Pz1OuWc14UfZtSUmJ+OyzzwQAsWnTpirTjx07VgwYMEAIIcTVq1cFAJGWllZlWmNjY5GQkFDlNG338QcffCBNV6vVolGjRmLZsmVCCCGWLVsm6tevL6UXQoj//ve/Gue9WbNmiXfeeUdjOefOnRMARE5OjhCi7NzVtm3bKuN8VVX8fzEyMhIDBw4Un3zyiWjevLlQq9VS2ri4OKFUKoVKpRKFhYVCT09PrF69Wpp+9+5dYWlpKebNmyeE+Oc8Uf57p7Lt3bdvXyFE2XG+bds2oVAoRFhYmBBC8/+1XN++fYWfn580bGdnJ7y9vTXSDBo0SPTo0UMaBiC+//57IcTjXyM8TMX/CG1/Uw86v508eVIaFxcXJxo3biyEePQ5Z/r06UJXV1f89ddf0rgtW7YIHR0dkZ+fL4QQwtLSUsyZM0djvo4dO4qgoCAhxD/bZeHChRpp4uPjhamp6SPXiV4NrBElAuDh4YGMjAxkZGTgwIED8PLyQo8ePaRmKvfLyMiAm5ubVCNRkb+/P06ePCk1E0xISICvr69Ug1NeI6qN3377DSqVCo6OjlAqldJn165dVTbzobLnKbX1888/4+2334aVlRWMjY0xbNgwXL16FUVFRVIauVxeqTayotdff13aNz169HjiuCt2TNGhQ4eHzjN06FCkpaXhwoULAIDVq1fj3XfflWoGMzMzkZCQoHHseHl5Qa1W4/Tp01ov53nxvO/bKVOmQKlUwtDQENHR0Zg7dy7effddAEBcXBzat2+Phg0bQqlUYsWKFVJT4Hr16sHf3x9eXl7o3bs3Fi1apNGEbuLEiRg5ciQ8PT0xd+5cjd++tvv4/vWUyWQwNzfHpUuXAJTVqDk7O0NfX19K06lTJ411y8zMxM6dOzWW06JFCwDQiKd9+/YP3Uavovv/XzIyMrB48WJkZWXBxcVF4zfv6uqKwsJC/PXXXzh16hRKS0vh6uoqTdfT00OnTp2QlZVVG6vxwti0aROUSiX09fXRo0cPDBo06LGbf7q4uFQa1na7P+oa4XFV9ZvS5vxmaGiIpk2bSsMWFhbSb/5R5xwAsLW1hZWVlTTs4uICtVqNnJwc3Lx5ExcuXNA4PoGyY7jidnpR/l+odrD3FSIARkZGcHBwkIa//PJLmJqa4r///S9GjhypkdbAwOCheTVq1Ai9e/dGfHw8mjRpgi1btmg80/mo+e9XWFgIXV1dHD58GLq6uhrTlEql1vm8Spo1awaZTIbs7OyHpjtz5gx69eqFwMBAzJkzB/Xq1cPu3bsxYsQI3L17F4aGhgDK9tejei788ccfpU4YHmf/lrt69SouX75cqQOTqpqf3q9jx45o2rQp1q5di8DAQHz//fdISEiQphcWFuKjjz5CSEhIpXltbW21Xs7z4nnftx9//DH8/f2lZyjL8167di3CwsIQGxsLFxcXGBsbY/78+di/f780b3x8PEJCQpCamoqkpCR8+umn2LZtG958801ERkbi/fffx+bNm7FlyxZMnz4da9euRb9+/bTexxUvimUy2WN1zlRYWIjevXsjOjq60jQLCwvp+4tyLNWkiv8v9Gx5eHhg2bJlkMvlsLS01OhESEdHp9INreruQOdJ/gMepuJvStvzW1W/+fvX/WHnnGcZP9H9WBAlqoJMJoOOjg6Ki4srTXN2dkZiYiJKS0sfeMdz5MiRGDJkCKytrdG0aVONu4bOzs7Yvn07ZsyY8cg42rZtC5VKhUuXLml0dkQPVq9ePXh5eSEuLg4hISGV/gRv3LgBMzMzHD58GGq1GrGxsdDRKWscUv7M3uOys7N7qpgXLVoEHR0d+Pj4PPa8Q4cOxerVq2FtbQ0dHR2pBg4A2rVrhz/++OOluQh+3vdtgwYNqtzW6enp6Ny5s8bzVVW1aGjbti3atm2LiIgIuLi4YM2aNdJFoaOjIxwdHTFhwgQMGTIE8fHx6NevX7Xs4+bNm+Prr79GSUkJFAoFAEid4pRr164dkpOTYW9vzx7Eq4GTkxOSk5M1WkKkp6fD2NgY1tbWqF+/vvRcYPkxWFpaioMHD0qd7cjlcgB4YP8Cr6qHFfwbNmyoUfOnUqnw+++/Sx0/lbu/47PyYScnJ62Wr801wtOozvPbw845eXl5uHDhAiwtLQGUbQMdHR00b94cJiYmsLS0RHp6Otzd3aX80tPTK7WmqEgul/OYJQmb5hKhrHfCixcv4uLFi8jKysK4ceOkGoCKgoODcfPmTQwePBiHDh1Cbm4uVq1aJXX8AQBeXl4wMTHB7Nmz8eGHH2rMHxERgYMHDyIoKAjHjh1DdnY2li1bptH7ZDlHR0cMHToUw4cPx3fffYfTp0/jwIEDiIqKwubNm6t/Q7wk4uLioFKp0KlTJyQnJyM3NxdZWVlYvHix1OTKwcEBpaWlWLJkCf7880+sWrWqRjpXuXXrFi5evIhz587hl19+wejRozF79mzMmTPniQoTQ4cOxZEjRzBnzhwMHDhQKkgAZU1F9+zZg+DgYGRkZCA3Nxc//PBDpY5sXiTP8759kGbNmuHQoUPYunUrTpw4galTp2oU9E6fPo2IiAjs3bsXZ8+exU8//YTc3Fw4OTmhuLgYwcHBSEtLw9mzZ5Geno6DBw9KF8XVsY/ff/99qNVqjB49GllZWdi6dStiYmIAQCokjR07FteuXcOQIUNw8OBBnDp1Clu3bsWHH37Ii8onEBQUhHPnzmHcuHHIzs7GDz/8gOnTp2PixInQ0dGBkZERAgMD8fHHHyM1NRV//PEHRo0ahaKiIowYMQJA2U0SmUyGTZs24fLlyxq9plLVunXrhs2bN2Pz5s3Izs5GYGBgle9hTU9Px7x583DixAnExcVh/fr1CA0N1WoZ2lwjPI3qOL897JxTTl9fH35+fsjMzMSvv/6KkJAQ+Pr6wtzcHEBZC5Do6GgkJSUhJycH4eHhyMjIeOR2sre3R2FhIbZv344rV65oNCemV1BtPqBK9Dzw8/MTAKSPsbGx6Nixo/j222+FEJU7IhBCiMzMTPHOO+8IQ0NDYWxsLNzc3MSpU6c08p06darQ1dUVFy5cqLTMtLQ00blzZ6FQKISZmZnw8vKSOpyo2JnC3bt3xbRp04S9vb3Q09MTFhYWol+/fuLYsWPVvi1eJhcuXBBjx44VdnZ2Qi6XCysrK9GnTx+xc+dOKc2CBQuEhYWFMDAwEF5eXmLlypUanX9Ud6cKdnZ20nEml8uFra2t8PX1FTt27NBI96BOSCp2VlSuU6dOAkClfIQQ4sCBA6J79+5CqVQKIyMj4ezsrNHBxMM6UHpePa/79kHb8c6dO8Lf31+YmpoKMzMzERgYKMLDw6V9efHiReHj4yMsLCyEXC4XdnZ2Ytq0aUKlUomSkhIxePBgYWNjI+RyubC0tBTBwcEaHQs9yT5u3bq1mD59ujScnp4unJ2dhVwuF+3btxdr1qwRAER2draU5sSJE6Jfv37CzMxMGBgYiBYtWojx48dLHe5U1RHMq+7+znMqSktLEx07dhRyuVyYm5uLKVOmiNLSUml6cXGxGDdunGjQoIFQKBTC1dVVHDhwQCOPmTNnCnNzcyGTyTQ63HlVPWx7C1H2fxoYGCjq1asnGjVqJKKioqrsrGjGjBnivffeE4aGhsLc3FwsWrRIIx88pLMiIbS7RqhKVZ0VVfWbepLz2/fffy/KL/sfds4R4p//mqVLlwpLS0uhr68vBg4cKK5duyblp1KpRGRkpLCyshJ6enqidevWYsuWLdL0qrZLuTFjxoj69esLABrnIXr1yIR4jN4fiEhrI0aMwOXLl6X3lxERvSjuf99fdT/zRkTPt8jISKSkpGi8G5XoWeCDHkTVrKCgAL/99hvWrFnDQigRvRBWrlyJ1157DVZWVsjMzMSUKVPg6+vLQigRET0zLIgSVbO+ffviwIEDGDNmDLp3717b4RARPdLFixcxbdo0XLx4ERYWFnjvvfcwZ86c2g6LiIheYmyaS0RERERERDWKveYSERERERFRjWJBlIiIiIiIiGoUC6JERERERERUo1gQJSIiIiIiohrFgigRERERERHVKBZEiYiI/p+/vz98fHyk4a5du2L8+PE1HkdaWhpkMhlu3LjxTJcjk8mQkpLyTJdBRERUFRZEiYjouebv7w+ZTAaZTAa5XA4HBwfMnDkT9+7de+bL/u677zBr1iyt0tZU4fHu3bto0KAB5s6dW+X0WbNmoXHjxigtLX2mcRARET0NFkSJiOi55+3tjfz8fOTm5mLSpEmIjIzE/Pnzq0x79+7daltuvXr1YGxsXG35VQe5XI4PPvgA8fHxlaYJIZCQkIDhw4dDT0+vFqIjIiLSDguiRET03FMoFDA3N4ednR0CAwPh6emJDRs2APinOe2cOXNgaWmJ5s2bAwDOnTsHX19fmJmZoV69eujbty/OnDkj5alSqTBx4kSYmZmhfv36mDx5MoQQGsut2DS3pKQEU6ZMgY2NDRQKBRwcHPDVV1/hzJkz8PDwAADUrVsXMpkM/v7+AAC1Wo2oqCg0adIEBgYGaN26Nb799luN5fz4449wdHSEgYEBPDw8NOKsyogRI3DixAns3r1bY/yuXbvw559/YsSIETh48CC6d++OBg0awNTUFO7u7jhy5MgD86yqRjcjIwMymUwjnt27d8PNzQ0GBgawsbFBSEgIbt++LU1funQpmjVrBn19fTRu3BgDBw586LoQEdGriQVRIiJ64RgYGGjUfG7fvh05OTnYtm0bNm3ahNLSUnh5ecHY2Bi//vor0tPToVQq4e3tLc0XGxuLhIQE/O9//8Pu3btx7do1fP/99w9d7vDhw/HNN99g8eLFyMrKwhdffAGlUgkbGxskJycDAHJycpCfn49FixYBAKKiorBy5UosX74cx48fx4QJE/DBBx9g165dAMoKzP3790fv3r2RkZGBkSNHIjw8/KFxtGrVCh07dsT//vc/jfHx8fHo3LkzWrRogVu3bsHPzw+7d+/Gvn370KxZM/Ts2RO3bt16vI19n1OnTsHb2xsDBgzAsWPHkJSUhN27dyM4OBgAcOjQIYSEhGDmzJnIyclBamoqunTp8sTLIyKil5ggIiJ6jvn5+Ym+ffsKIYRQq9Vi27ZtQqFQiLCwMGl648aNRUlJiTTPqlWrRPPmzYVarZbGlZSUCAMDA7F161YhhBAWFhZi3rx50vTS0lJhbW0tLUsIIdzd3UVoaKgQQoicnBwBQGzbtq3KOHfu3CkAiOvXr0vj7ty5IwwNDcWePXs00o4YMUIMGTJECCFERESEaNmypcb0KVOmVMqrouXLlwulUilu3bolhBDi5s2bwtDQUHz55ZdVplepVMLY2Fhs3LhRGgdAfP/99w+M/+jRowKAOH36tBT36NGjNfL99ddfhY6OjiguLhbJycnCxMRE3Lx584FxExERCSEEa0SJiOi5t2nTJiiVSujr66NHjx4YNGgQIiMjpemtWrWCXC6XhjMzM3Hy5EkYGxtDqVRCqVSiXr16uHPnDk6dOoWCggLk5+fjjTfekOapU6cOOnTo8MAYMjIyoKurC3d3d63jPnnyJIqKitC9e3cpDqVSiZUrV+LUqVMAgKysLI04AMDFxeWReQ8ZMgQqlQrr1q0DACQlJUFHRweDBg0CAPz9998YNWoUmjVrBlNTU5iYmKCwsBB5eXlax19RZmYmEhISNNbFy8sLarUap0+fRvfu3WFnZ4fXXnsNw4YNw+rVq1FUVPTEyyMiopdXndoOgIiI6FE8PDywbNkyyOVyWFpaok4dzb8vIyMjjeHCwkK0b98eq1evrpRXw4YNnygGAwODx56nsLAQALB582ZYWVlpTFMoFE8URzkTExMMHDgQ8fHxCAgIQHx8PHx9faFUKgEAfn5+uHr1KhYtWgQ7OzsoFAq4uLg8sDMnHZ2ye9PivudkK/a8W1hYiI8++gghISGV5re1tYVcLseRI0eQlpaGn376CdOmTUNkZCQOHjwIMzOzp1pfIiJ6ubAgSkREzz0jIyM4ODhonb5du3ZISkpCo0aNYGJiUmUaCwsL7N+/X3qG8d69ezh8+DDatWtXZfpWrVpBrVZj165d8PT0rDS9vEZWpVJJ41q2bAmFQoG8vLwH1qQ6OTlJHS+V27dv36NXEmWdFnXt2hWbNm3Cnj17NHoSTk9Px9KlS9GzZ08AZc+iXrly5YF5lRfQ8/PzUbduXQBltcD3a9euHf7444+H7os6derA09MTnp6emD59OszMzLBjxw70799fq3UiIqJXA5vmEhHRS2fo0KFo0KAB+vbti19//RWnT59GWloaQkJC8NdffwEAQkNDMXfuXKSkpCA7OxtBQUEPfQeovb09/Pz8EBAQgJSUFCnP8qaxdnZ2kMlk2LRpEy5fvozCwkIYGxsjLCwMEyZMQGJiIk6dOoUjR45gyZIlSExMBACMGTMGubm5+Pjjj5GTk4M1a9YgISFBq/Xs0qULHBwcMHz4cLRo0QKdO3eWpjVr1gyrVq1CVlYW9u/fj6FDhz60VtfBwQE2NjaIjIxEbm4uNm/ejNjYWI00U6ZMwZ49exAcHIyMjAzk5ubihx9+kDor2rRpExYvXoyMjAycPXsWK1euhFqtlnoyJiIiKseCKBERvXQMDQ3xyy+/wNbWFv3794eTkxNGjBiBO3fuSDWkkyZNwrBhw+Dn5wcXFxcYGxujX79+D8132bJlGDhwIIKCgtCiRQuMGjVKenWJlZUVZsyYgfDwcDRu3FgqnM2aNQtTp05FVFQUnJyc4O3tjc2bN6NJkyYAypq0JicnIyUlBa1bt8by5cvx2WefabWeMpkMAQEBuH79OgICAjSmffXVV7h+/TratWuHYcOGISQkBI0aNXpgXnp6evjmm2+QnZ0NZ2dnREdHY/bs2RppnJ2dsWvXLpw4cQJubm5o27Ytpk2bBktLSwCAmZkZvvvuO3Tr1g1OTk5Yvnw5vvnmG7z++utarQ8REb06ZEJUeGkaERERERER0TPEGlEiIiIiIiKqUSyIEhERERERUY1iQZSIiIiIiIhqFAuiREREREREVKNYECUiIiIiIqIaxYIoERERERER1SgWRImIiIiIiKhGsSBKRERERERENYoFUSIiIiIiIqpRLIgSERERERFRjWJBlIiIiIiIiGoUC6JERERERERUo/4P0jbegwGy4uQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1000x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mapping = {'Bicycle':0, 'Car - Driver':1, 'Car - Passenger':2, 'Foot':3,\n",
    "       'Public Transport':4}\n",
    "def conf_matrix(y_true, y_pred, title, ax):\n",
    "    cm = pd.DataFrame(confusion_matrix(y_true, y_pred), index=mapping.keys(), columns=mapping.keys())\n",
    "    sns.heatmap(cm, annot=True, fmt='g', cmap=\"Blues\", ax=ax)\n",
    "    ax.set_title(title)\n",
    "    ax.set_ylabel('Actual Values')\n",
    "    ax.set_xlabel('Predicted Values')\n",
    "    \n",
    "\n",
    "fig, axes = plt.subplots(1, 1, figsize=(10, 5)) \n",
    "\n",
    "conf_matrix(y_test, y_pred_xgb_10000, title=\"Confusion Matrix: xgb_dist10000\", ax=axes)  \n",
    "plt.tight_layout()  \n",
    "plt.savefig(\"confusion_matrices_dist10000.pdf\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "4c137ea1-8712-499c-9584-3b6e80d5b956",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Precision (Macro)</th>\n",
       "      <th>Recall (Macro)</th>\n",
       "      <th>F1-score (Macro)</th>\n",
       "      <th>Balanced Accuracy</th>\n",
       "      <th>Log Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>KNN Rook</th>\n",
       "      <td>73.76%</td>\n",
       "      <td>56.47%</td>\n",
       "      <td>57.96%</td>\n",
       "      <td>56.47%</td>\n",
       "      <td>0.9117440860445595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNN KNN8</th>\n",
       "      <td>73.73%</td>\n",
       "      <td>57.21%</td>\n",
       "      <td>58.73%</td>\n",
       "      <td>57.21%</td>\n",
       "      <td>0.9028393960479246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KNN dist10000</th>\n",
       "      <td>73.89%</td>\n",
       "      <td>57.15%</td>\n",
       "      <td>58.71%</td>\n",
       "      <td>57.15%</td>\n",
       "      <td>0.8990080862772643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Random Forest Rook</th>\n",
       "      <td>78.03%</td>\n",
       "      <td>73.74%</td>\n",
       "      <td>74.95%</td>\n",
       "      <td>73.74%</td>\n",
       "      <td>0.613405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Random Forest KNN8</th>\n",
       "      <td>78.4%</td>\n",
       "      <td>73.77%</td>\n",
       "      <td>74.96%</td>\n",
       "      <td>73.77%</td>\n",
       "      <td>0.591361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Random Forest dist10000</th>\n",
       "      <td>78.18%</td>\n",
       "      <td>73.47%</td>\n",
       "      <td>74.86%</td>\n",
       "      <td>73.47%</td>\n",
       "      <td>0.602151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBoost Rook</th>\n",
       "      <td>79.27%</td>\n",
       "      <td>75.58%</td>\n",
       "      <td>76.8%</td>\n",
       "      <td>75.58%</td>\n",
       "      <td>0.543022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBoost KNN8</th>\n",
       "      <td>79.28%</td>\n",
       "      <td>75.61%</td>\n",
       "      <td>76.81%</td>\n",
       "      <td>75.61%</td>\n",
       "      <td>0.542075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBoost dist10000</th>\n",
       "      <td>79.39%</td>\n",
       "      <td>75.74%</td>\n",
       "      <td>77.0%</td>\n",
       "      <td>75.74%</td>\n",
       "      <td>0.539766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TabNet Rook</th>\n",
       "      <td>76.5%</td>\n",
       "      <td>71.65%</td>\n",
       "      <td>73.32%</td>\n",
       "      <td>71.65%</td>\n",
       "      <td>0.628897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TabNet KNN8</th>\n",
       "      <td>77.67%</td>\n",
       "      <td>73.18%</td>\n",
       "      <td>74.72%</td>\n",
       "      <td>73.18%</td>\n",
       "      <td>0.598817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TabNet dist10000</th>\n",
       "      <td>77.94%</td>\n",
       "      <td>73.33%</td>\n",
       "      <td>74.88%</td>\n",
       "      <td>73.33%</td>\n",
       "      <td>0.593555</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        Precision (Macro) Recall (Macro) F1-score (Macro)  \\\n",
       "KNN Rook                           73.76%         56.47%           57.96%   \n",
       "KNN KNN8                           73.73%         57.21%           58.73%   \n",
       "KNN dist10000                      73.89%         57.15%           58.71%   \n",
       "Random Forest Rook                 78.03%         73.74%           74.95%   \n",
       "Random Forest KNN8                  78.4%         73.77%           74.96%   \n",
       "Random Forest dist10000            78.18%         73.47%           74.86%   \n",
       "XGBoost Rook                       79.27%         75.58%            76.8%   \n",
       "XGBoost KNN8                       79.28%         75.61%           76.81%   \n",
       "XGBoost dist10000                  79.39%         75.74%            77.0%   \n",
       "TabNet Rook                         76.5%         71.65%           73.32%   \n",
       "TabNet KNN8                        77.67%         73.18%           74.72%   \n",
       "TabNet dist10000                   77.94%         73.33%           74.88%   \n",
       "\n",
       "                        Balanced Accuracy            Log Loss  \n",
       "KNN Rook                           56.47%  0.9117440860445595  \n",
       "KNN KNN8                           57.21%  0.9028393960479246  \n",
       "KNN dist10000                      57.15%  0.8990080862772643  \n",
       "Random Forest Rook                 73.74%            0.613405  \n",
       "Random Forest KNN8                 73.77%            0.591361  \n",
       "Random Forest dist10000            73.47%            0.602151  \n",
       "XGBoost Rook                       75.58%            0.543022  \n",
       "XGBoost KNN8                       75.61%            0.542075  \n",
       "XGBoost dist10000                  75.74%            0.539766  \n",
       "TabNet Rook                        71.65%            0.628897  \n",
       "TabNet KNN8                        73.18%            0.598817  \n",
       "TabNet dist10000                   73.33%            0.593555  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_metrics_spatial = pd.concat([combined_metrics_df_knn, combined_metrics_df_rf,combined_metrics_df_xgb, combined_metrics_df_tabnet], axis=1)\n",
    "combined_metrics_spatial.to_csv(\"combined_metrics_spatial.csv\")\n",
    "combined_metrics_spatial.T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1ff336e-0639-4929-8ed6-be6c9efd4740",
   "metadata": {},
   "source": [
    "<a id='section9'></a>\n",
    "# # Province analysis (error count)\n",
    "[Back to Table of Contents](#Table-of-Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "c3707280-e2e6-41bb-8ade-9d32876e7eab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load provinces data\n",
    "provinces = pd.read_csv(\"X_test_region.csv\")\n",
    "provinces = provinces[[\"ResidentialProvince\"]]\n",
    "\n",
    "# Load predictions\n",
    "y_pred = pd.DataFrame(y_pred_xgb_10000)\n",
    "y_pred.rename(columns={0: \"y_pred\"}, inplace=True)\n",
    "predicted_probs = pd.DataFrame(y_pred_prob_xgb_10000)\n",
    "\n",
    "# y_test\n",
    "y_test = pd.DataFrame(y_test)\n",
    "y_test.rename(columns={0: \"y_test\"}, inplace=True)\n",
    "\n",
    "# Combine the results\n",
    "df_prediction = pd.concat([provinces, y_test, y_pred, predicted_probs], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "e6ebf646-3d79-434c-85be-30070be7f5ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ResidentialProvince</th>\n",
       "      <th>Wrong Predicted Car - Driver</th>\n",
       "      <th>Wrong Predicted Bicycle</th>\n",
       "      <th>Wrong Predicted Car - Passenger</th>\n",
       "      <th>Wrong Predicted Foot</th>\n",
       "      <th>Wrong Predicted Public Transport</th>\n",
       "      <th>Total Observations per Region</th>\n",
       "      <th>% of Wrong Predicted Car - Driver</th>\n",
       "      <th>% of Wrong Predicted Bicycle</th>\n",
       "      <th>% of Wrong Predicted Car - Passenger</th>\n",
       "      <th>% of Wrong Predicted Foot</th>\n",
       "      <th>% of Wrong Predicted Public Transport</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Fryslân</td>\n",
       "      <td>89</td>\n",
       "      <td>74</td>\n",
       "      <td>68</td>\n",
       "      <td>43</td>\n",
       "      <td>12</td>\n",
       "      <td>1180</td>\n",
       "      <td>7.54</td>\n",
       "      <td>6.27</td>\n",
       "      <td>5.76</td>\n",
       "      <td>3.64</td>\n",
       "      <td>1.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Noord-Holland</td>\n",
       "      <td>252</td>\n",
       "      <td>273</td>\n",
       "      <td>232</td>\n",
       "      <td>121</td>\n",
       "      <td>32</td>\n",
       "      <td>3929</td>\n",
       "      <td>6.41</td>\n",
       "      <td>6.95</td>\n",
       "      <td>5.90</td>\n",
       "      <td>3.08</td>\n",
       "      <td>0.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Noord-Brabant</td>\n",
       "      <td>250</td>\n",
       "      <td>339</td>\n",
       "      <td>318</td>\n",
       "      <td>119</td>\n",
       "      <td>41</td>\n",
       "      <td>4672</td>\n",
       "      <td>5.35</td>\n",
       "      <td>7.26</td>\n",
       "      <td>6.81</td>\n",
       "      <td>2.55</td>\n",
       "      <td>0.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Zuid-Holland</td>\n",
       "      <td>176</td>\n",
       "      <td>306</td>\n",
       "      <td>207</td>\n",
       "      <td>128</td>\n",
       "      <td>47</td>\n",
       "      <td>4127</td>\n",
       "      <td>4.26</td>\n",
       "      <td>7.41</td>\n",
       "      <td>5.02</td>\n",
       "      <td>3.10</td>\n",
       "      <td>1.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Overijssel</td>\n",
       "      <td>82</td>\n",
       "      <td>79</td>\n",
       "      <td>64</td>\n",
       "      <td>47</td>\n",
       "      <td>3</td>\n",
       "      <td>1416</td>\n",
       "      <td>5.79</td>\n",
       "      <td>5.58</td>\n",
       "      <td>4.52</td>\n",
       "      <td>3.32</td>\n",
       "      <td>0.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Utrecht</td>\n",
       "      <td>363</td>\n",
       "      <td>441</td>\n",
       "      <td>301</td>\n",
       "      <td>213</td>\n",
       "      <td>46</td>\n",
       "      <td>6294</td>\n",
       "      <td>5.77</td>\n",
       "      <td>7.01</td>\n",
       "      <td>4.78</td>\n",
       "      <td>3.38</td>\n",
       "      <td>0.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Limburg</td>\n",
       "      <td>102</td>\n",
       "      <td>195</td>\n",
       "      <td>166</td>\n",
       "      <td>107</td>\n",
       "      <td>27</td>\n",
       "      <td>2898</td>\n",
       "      <td>3.52</td>\n",
       "      <td>6.73</td>\n",
       "      <td>5.73</td>\n",
       "      <td>3.69</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Gelderland</td>\n",
       "      <td>304</td>\n",
       "      <td>393</td>\n",
       "      <td>292</td>\n",
       "      <td>151</td>\n",
       "      <td>49</td>\n",
       "      <td>5249</td>\n",
       "      <td>5.79</td>\n",
       "      <td>7.49</td>\n",
       "      <td>5.56</td>\n",
       "      <td>2.88</td>\n",
       "      <td>0.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Flevoland</td>\n",
       "      <td>24</td>\n",
       "      <td>39</td>\n",
       "      <td>49</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "      <td>721</td>\n",
       "      <td>3.33</td>\n",
       "      <td>5.41</td>\n",
       "      <td>6.80</td>\n",
       "      <td>4.30</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>Drenthe</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>19</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>538</td>\n",
       "      <td>4.09</td>\n",
       "      <td>3.72</td>\n",
       "      <td>3.53</td>\n",
       "      <td>2.04</td>\n",
       "      <td>0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>Groningen</td>\n",
       "      <td>124</td>\n",
       "      <td>124</td>\n",
       "      <td>180</td>\n",
       "      <td>86</td>\n",
       "      <td>24</td>\n",
       "      <td>2623</td>\n",
       "      <td>4.73</td>\n",
       "      <td>4.73</td>\n",
       "      <td>6.86</td>\n",
       "      <td>3.28</td>\n",
       "      <td>0.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>Zeeland</td>\n",
       "      <td>36</td>\n",
       "      <td>63</td>\n",
       "      <td>18</td>\n",
       "      <td>26</td>\n",
       "      <td>7</td>\n",
       "      <td>635</td>\n",
       "      <td>5.67</td>\n",
       "      <td>9.92</td>\n",
       "      <td>2.83</td>\n",
       "      <td>4.09</td>\n",
       "      <td>1.10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    ResidentialProvince  Wrong Predicted Car - Driver  \\\n",
       "0               Fryslân                            89   \n",
       "6         Noord-Holland                           252   \n",
       "8         Noord-Brabant                           250   \n",
       "10         Zuid-Holland                           176   \n",
       "23           Overijssel                            82   \n",
       "25              Utrecht                           363   \n",
       "27              Limburg                           102   \n",
       "51           Gelderland                           304   \n",
       "56            Flevoland                            24   \n",
       "187             Drenthe                            22   \n",
       "263           Groningen                           124   \n",
       "299             Zeeland                            36   \n",
       "\n",
       "     Wrong Predicted Bicycle  Wrong Predicted Car - Passenger  \\\n",
       "0                         74                               68   \n",
       "6                        273                              232   \n",
       "8                        339                              318   \n",
       "10                       306                              207   \n",
       "23                        79                               64   \n",
       "25                       441                              301   \n",
       "27                       195                              166   \n",
       "51                       393                              292   \n",
       "56                        39                               49   \n",
       "187                       20                               19   \n",
       "263                      124                              180   \n",
       "299                       63                               18   \n",
       "\n",
       "     Wrong Predicted Foot  Wrong Predicted Public Transport  \\\n",
       "0                      43                                12   \n",
       "6                     121                                32   \n",
       "8                     119                                41   \n",
       "10                    128                                47   \n",
       "23                     47                                 3   \n",
       "25                    213                                46   \n",
       "27                    107                                27   \n",
       "51                    151                                49   \n",
       "56                     31                                 0   \n",
       "187                    11                                 1   \n",
       "263                    86                                24   \n",
       "299                    26                                 7   \n",
       "\n",
       "     Total Observations per Region  % of Wrong Predicted Car - Driver  \\\n",
       "0                             1180                               7.54   \n",
       "6                             3929                               6.41   \n",
       "8                             4672                               5.35   \n",
       "10                            4127                               4.26   \n",
       "23                            1416                               5.79   \n",
       "25                            6294                               5.77   \n",
       "27                            2898                               3.52   \n",
       "51                            5249                               5.79   \n",
       "56                             721                               3.33   \n",
       "187                            538                               4.09   \n",
       "263                           2623                               4.73   \n",
       "299                            635                               5.67   \n",
       "\n",
       "     % of Wrong Predicted Bicycle  % of Wrong Predicted Car - Passenger  \\\n",
       "0                            6.27                                  5.76   \n",
       "6                            6.95                                  5.90   \n",
       "8                            7.26                                  6.81   \n",
       "10                           7.41                                  5.02   \n",
       "23                           5.58                                  4.52   \n",
       "25                           7.01                                  4.78   \n",
       "27                           6.73                                  5.73   \n",
       "51                           7.49                                  5.56   \n",
       "56                           5.41                                  6.80   \n",
       "187                          3.72                                  3.53   \n",
       "263                          4.73                                  6.86   \n",
       "299                          9.92                                  2.83   \n",
       "\n",
       "     % of Wrong Predicted Foot  % of Wrong Predicted Public Transport  \n",
       "0                         3.64                                   1.02  \n",
       "6                         3.08                                   0.81  \n",
       "8                         2.55                                   0.88  \n",
       "10                        3.10                                   1.14  \n",
       "23                        3.32                                   0.21  \n",
       "25                        3.38                                   0.73  \n",
       "27                        3.69                                   0.93  \n",
       "51                        2.88                                   0.93  \n",
       "56                        4.30                                   0.00  \n",
       "187                       2.04                                   0.19  \n",
       "263                       3.28                                   0.91  \n",
       "299                       4.09                                   1.10  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observations_per_region = df_prediction.groupby('ResidentialProvince').size().reset_index(name='observations_per_region')\n",
    "df_prediction = pd.merge(df_prediction, observations_per_region, on='ResidentialProvince', how='left')\n",
    "\n",
    "# count number of incorrect classified per province per mdoe \n",
    "df_prediction['is_class1'] = (df_prediction[\"y_test\"] == 1) & (df_prediction[\"y_pred\"] != 1)\n",
    "df_prediction['is_class2'] = (df_prediction[\"y_test\"] == 2) & (df_prediction[\"y_pred\"] != 2)\n",
    "df_prediction['is_class3'] = (df_prediction[\"y_test\"] == 3) & (df_prediction[\"y_pred\"] != 3)\n",
    "df_prediction['is_class4'] = (df_prediction[\"y_test\"] == 4) & (df_prediction[\"y_pred\"] != 4)\n",
    "df_prediction['is_class0'] = (df_prediction[\"y_test\"] == 0) & (df_prediction[\"y_pred\"] != 0)\n",
    "\n",
    "\n",
    "incorrect_class1_counts = df_prediction.groupby('ResidentialProvince')['is_class1'].sum()\n",
    "incorrect_class0_counts = df_prediction.groupby('ResidentialProvince')['is_class0'].sum()\n",
    "incorrect_class2_counts = df_prediction.groupby('ResidentialProvince')['is_class2'].sum()\n",
    "incorrect_class3_counts = df_prediction.groupby('ResidentialProvince')['is_class3'].sum()\n",
    "incorrect_class4_counts = df_prediction.groupby('ResidentialProvince')['is_class4'].sum()\n",
    "\n",
    "\n",
    "incorrect_counts_df = pd.DataFrame({\n",
    "    'incorrect_class1_counts': incorrect_class1_counts,\n",
    "    'incorrect_class0_counts': incorrect_class0_counts,\n",
    "    'incorrect_class2_counts': incorrect_class2_counts,\n",
    "    'incorrect_class3_counts': incorrect_class3_counts,\n",
    "    'incorrect_class4_counts': incorrect_class4_counts\n",
    "}).reset_index()\n",
    "df_prediction_per_mode = pd.merge(df_prediction, incorrect_counts_df, on='ResidentialProvince', how='left')\n",
    "df_prediction_per_mode.drop_duplicates(\"ResidentialProvince\", inplace=True)\n",
    "\n",
    "df_prediction_per_mode = df_prediction_per_mode[[\"ResidentialProvince\", \"incorrect_class1_counts\", 'incorrect_class0_counts','incorrect_class2_counts','incorrect_class3_counts',\n",
    "                        'incorrect_class4_counts', \"observations_per_region\"]]\n",
    "\n",
    "\n",
    "df_prediction_per_mode.rename(columns = {\"incorrect_class1_counts\":'Car - Driver', 'incorrect_class0_counts':'Bicycle',\n",
    "                                         'incorrect_class2_counts':'Car - Passenger', \n",
    "                                         'incorrect_class3_counts':'Foot', 'incorrect_class4_counts':\"Public Transport\" }, inplace = True)\n",
    "\n",
    "# add % wrong predicted\n",
    "df_prediction_per_mode['%_Car_Driver'] = round((df_prediction_per_mode['Car - Driver'] / df_prediction_per_mode['observations_per_region']) * 100,2)\n",
    "df_prediction_per_mode['%_Bicycle'] = round((df_prediction_per_mode['Bicycle'] / df_prediction_per_mode['observations_per_region']) * 100,2)\n",
    "df_prediction_per_mode['%_Car_Passenger'] = round((df_prediction_per_mode['Car - Passenger'] / df_prediction_per_mode['observations_per_region']) * 100,2)\n",
    "df_prediction_per_mode['%_Foot'] = round((df_prediction_per_mode['Foot'] / df_prediction_per_mode['observations_per_region']) * 100,2)\n",
    "df_prediction_per_mode['%_Public_Transportr'] = round((df_prediction_per_mode['Public Transport'] / df_prediction_per_mode['observations_per_region']) * 100,2)\n",
    "\n",
    "df_prediction_per_mode.columns = ['ResidentialProvince', 'Wrong Predicted Car - Driver', 'Wrong Predicted Bicycle', \n",
    "    'Wrong Predicted Car - Passenger',  'Wrong Predicted Foot',   'Wrong Predicted Public Transport', \n",
    "    'Total Observations per Region',   '% of Wrong Predicted Car - Driver',   '% of Wrong Predicted Bicycle', \n",
    "    '% of Wrong Predicted Car - Passenger',  '% of Wrong Predicted Foot', '% of Wrong Predicted Public Transport']\n",
    "\n",
    "df_prediction_per_mode.to_csv(\"df_prediction_per_mode.csv\")\n",
    "df_prediction_per_mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "b0be9105-0255-4788-8c13-fb5bf246ab82",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Car - Driver</th>\n",
       "      <th>Bicycle</th>\n",
       "      <th>Car - Passenger</th>\n",
       "      <th>Foot</th>\n",
       "      <th>Public Transport</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Car - Driver</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.963017</td>\n",
       "      <td>0.936610</td>\n",
       "      <td>0.943949</td>\n",
       "      <td>0.892803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bicycle</th>\n",
       "      <td>0.963017</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.950997</td>\n",
       "      <td>0.963430</td>\n",
       "      <td>0.958459</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Car - Passenger</th>\n",
       "      <td>0.936610</td>\n",
       "      <td>0.950997</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.921702</td>\n",
       "      <td>0.940321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Foot</th>\n",
       "      <td>0.943949</td>\n",
       "      <td>0.963430</td>\n",
       "      <td>0.921702</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.923023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Public Transport</th>\n",
       "      <td>0.892803</td>\n",
       "      <td>0.958459</td>\n",
       "      <td>0.940321</td>\n",
       "      <td>0.923023</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Car - Driver   Bicycle  Car - Passenger      Foot  \\\n",
       "Car - Driver          1.000000  0.963017         0.936610  0.943949   \n",
       "Bicycle               0.963017  1.000000         0.950997  0.963430   \n",
       "Car - Passenger       0.936610  0.950997         1.000000  0.921702   \n",
       "Foot                  0.943949  0.963430         0.921702  1.000000   \n",
       "Public Transport      0.892803  0.958459         0.940321  0.923023   \n",
       "\n",
       "                  Public Transport  \n",
       "Car - Driver              0.892803  \n",
       "Bicycle                   0.958459  \n",
       "Car - Passenger           0.940321  \n",
       "Foot                      0.923023  \n",
       "Public Transport          1.000000  "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prediction_per_mode.set_index('ResidentialProvince', inplace=True)\n",
    "df_prediction_per_mode.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "2b64b3fc-bcea-43c3-806d-908880cadb8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count the number of observations per region\n",
    "observations_per_region = df_prediction.groupby('ResidentialProvince').size().reset_index(name='observations_per_region')\n",
    "df_prediction = pd.merge(df_prediction, observations_per_region, on='ResidentialProvince', how='left')\n",
    "\n",
    "\n",
    "# is the prediction correct? \n",
    "df_prediction['is_correct'] = (df_prediction['y_test'] != df_prediction['y_pred'])\n",
    "\n",
    "# Count wrong prediction per province\n",
    "wrong_per_region = df_prediction.groupby('ResidentialProvince')['is_correct'].sum().reset_index(name='wrong_observations_per_region')\n",
    "\n",
    "# Merge the counts back into the original DataFrame\n",
    "df_prediction = pd.merge(df_prediction, wrong_per_region, on='ResidentialProvince', how='left')\n",
    "\n",
    "# Calculate the percentage of wrong predictions per region\n",
    "df_prediction['wrong_predictions_percentage'] = round((df_prediction['wrong_observations_per_region'] / df_prediction['observations_per_region']) * 100,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "ab5ec128-7ea5-46dc-8c64-3900b45f8344",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sum probabilities across all classes\n",
    "df_prediction['aggregated_prob'] = df_prediction[[0, 1, 2, 3, 4]].mean(axis=1)\n",
    "\n",
    "# calculate standard error for aggregated probabilities within each region\n",
    "df_prediction['se'] = np.sqrt(df_prediction['aggregated_prob'] * (1 - df_prediction['aggregated_prob']) / df_prediction['observations_per_region'])\n",
    "\n",
    "# determine confidence level\n",
    "confidence_level = 0.95\n",
    "\n",
    "#  z-score for ci\n",
    "z_score = scipy.stats.norm.ppf((1 + confidence_level) / 2)\n",
    "\n",
    "# margin error\n",
    "df_prediction['margin_of_error'] = z_score * df_prediction['se']\n",
    "\n",
    "#  upper and lower bounds of the confidence interval for aggregated probabilities within each region\n",
    "df_prediction['lower_ci'] = df_prediction['aggregated_prob'] - df_prediction['margin_of_error']\n",
    "df_prediction['upper_ci'] = df_prediction['aggregated_prob'] + df_prediction['margin_of_error']\n",
    "\n",
    "\n",
    "# extract relevant columns \n",
    "df_prediction = df_prediction[[\"ResidentialProvince\", \"observations_per_region\", \"wrong_observations_per_region\", 'wrong_predictions_percentage', \n",
    "                              \"se\", \"margin_of_error\", \t\"lower_ci\", \"upper_ci\"]]\n",
    "# drop duplicates provinces\n",
    "df_prediction.drop_duplicates(\"ResidentialProvince\", inplace=True)\n",
    "\n",
    "# Save as df\n",
    "df_prediction.to_csv(\"df_prediction_with_ci.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "f906257e-76e8-44cf-a33d-9b246b920677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Province</th>\n",
       "      <th>N.of Trips</th>\n",
       "      <th>N.of Wrong Predicted Trips</th>\n",
       "      <th>% Predicted Wrong</th>\n",
       "      <th>se</th>\n",
       "      <th>margin_of_error</th>\n",
       "      <th>lower_ci</th>\n",
       "      <th>upper_ci</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Fryslân</td>\n",
       "      <td>1180</td>\n",
       "      <td>286</td>\n",
       "      <td>24.24</td>\n",
       "      <td>0.011644</td>\n",
       "      <td>0.022823</td>\n",
       "      <td>0.177177</td>\n",
       "      <td>0.222823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Noord-Holland</td>\n",
       "      <td>3929</td>\n",
       "      <td>910</td>\n",
       "      <td>23.16</td>\n",
       "      <td>0.006381</td>\n",
       "      <td>0.012507</td>\n",
       "      <td>0.187493</td>\n",
       "      <td>0.212507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Noord-Brabant</td>\n",
       "      <td>4672</td>\n",
       "      <td>1067</td>\n",
       "      <td>22.84</td>\n",
       "      <td>0.005852</td>\n",
       "      <td>0.011470</td>\n",
       "      <td>0.188530</td>\n",
       "      <td>0.211470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Zuid-Holland</td>\n",
       "      <td>4127</td>\n",
       "      <td>864</td>\n",
       "      <td>20.94</td>\n",
       "      <td>0.006226</td>\n",
       "      <td>0.012204</td>\n",
       "      <td>0.187796</td>\n",
       "      <td>0.212204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Overijssel</td>\n",
       "      <td>1416</td>\n",
       "      <td>275</td>\n",
       "      <td>19.42</td>\n",
       "      <td>0.010630</td>\n",
       "      <td>0.020834</td>\n",
       "      <td>0.179166</td>\n",
       "      <td>0.220834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Utrecht</td>\n",
       "      <td>6294</td>\n",
       "      <td>1364</td>\n",
       "      <td>21.67</td>\n",
       "      <td>0.005042</td>\n",
       "      <td>0.009882</td>\n",
       "      <td>0.190118</td>\n",
       "      <td>0.209882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Limburg</td>\n",
       "      <td>2898</td>\n",
       "      <td>597</td>\n",
       "      <td>20.60</td>\n",
       "      <td>0.007430</td>\n",
       "      <td>0.014563</td>\n",
       "      <td>0.185437</td>\n",
       "      <td>0.214563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Gelderland</td>\n",
       "      <td>5249</td>\n",
       "      <td>1189</td>\n",
       "      <td>22.65</td>\n",
       "      <td>0.005521</td>\n",
       "      <td>0.010821</td>\n",
       "      <td>0.189179</td>\n",
       "      <td>0.210821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Flevoland</td>\n",
       "      <td>721</td>\n",
       "      <td>143</td>\n",
       "      <td>19.83</td>\n",
       "      <td>0.014897</td>\n",
       "      <td>0.029197</td>\n",
       "      <td>0.170803</td>\n",
       "      <td>0.229197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>Drenthe</td>\n",
       "      <td>538</td>\n",
       "      <td>73</td>\n",
       "      <td>13.57</td>\n",
       "      <td>0.017245</td>\n",
       "      <td>0.033800</td>\n",
       "      <td>0.166200</td>\n",
       "      <td>0.233800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>Groningen</td>\n",
       "      <td>2623</td>\n",
       "      <td>538</td>\n",
       "      <td>20.51</td>\n",
       "      <td>0.007810</td>\n",
       "      <td>0.015308</td>\n",
       "      <td>0.184692</td>\n",
       "      <td>0.215308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>Zeeland</td>\n",
       "      <td>635</td>\n",
       "      <td>150</td>\n",
       "      <td>23.62</td>\n",
       "      <td>0.015874</td>\n",
       "      <td>0.031112</td>\n",
       "      <td>0.168888</td>\n",
       "      <td>0.231112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Province  N.of Trips  N.of Wrong Predicted Trips  % Predicted Wrong  \\\n",
       "0          Fryslân        1180                         286              24.24   \n",
       "6    Noord-Holland        3929                         910              23.16   \n",
       "8    Noord-Brabant        4672                        1067              22.84   \n",
       "10    Zuid-Holland        4127                         864              20.94   \n",
       "23      Overijssel        1416                         275              19.42   \n",
       "25         Utrecht        6294                        1364              21.67   \n",
       "27         Limburg        2898                         597              20.60   \n",
       "51      Gelderland        5249                        1189              22.65   \n",
       "56       Flevoland         721                         143              19.83   \n",
       "187        Drenthe         538                          73              13.57   \n",
       "263      Groningen        2623                         538              20.51   \n",
       "299        Zeeland         635                         150              23.62   \n",
       "\n",
       "           se  margin_of_error  lower_ci  upper_ci  \n",
       "0    0.011644         0.022823  0.177177  0.222823  \n",
       "6    0.006381         0.012507  0.187493  0.212507  \n",
       "8    0.005852         0.011470  0.188530  0.211470  \n",
       "10   0.006226         0.012204  0.187796  0.212204  \n",
       "23   0.010630         0.020834  0.179166  0.220834  \n",
       "25   0.005042         0.009882  0.190118  0.209882  \n",
       "27   0.007430         0.014563  0.185437  0.214563  \n",
       "51   0.005521         0.010821  0.189179  0.210821  \n",
       "56   0.014897         0.029197  0.170803  0.229197  \n",
       "187  0.017245         0.033800  0.166200  0.233800  \n",
       "263  0.007810         0.015308  0.184692  0.215308  \n",
       "299  0.015874         0.031112  0.168888  0.231112  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prediction.rename(columns={\"ResidentialProvince\": \"Province\", \"observations_per_region\":\"N.of Trips\", \n",
    "                            \"wrong_observations_per_region\":\"N.of Wrong Predicted Trips\", \n",
    "                             \"wrong_predictions_percentage\": \"% Predicted Wrong\"}, inplace = True)\n",
    "df_prediction"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}