5-comparing-with-related-work.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "\n",
    "import sys, os\n",
    "import sklearn\n",
    "import datetime\n",
    "\n",
    "import importlib\n",
    "from tqdm import tqdm\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<module 'nn_evaluation' from '/Users/andrewt/Documents/Univ. Piraeus Research Center/VCRA/nn_evaluation.py'>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import cri_calc as cri\n",
    "importlib.reload(cri)\n",
    "\n",
    "import cri_helper as helper\n",
    "importlib.reload(helper)\n",
    "\n",
    "import nn_evaluation as nne\n",
    "importlib.reload(nne)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, mean_squared_log_error, mean_absolute_percentage_error\n",
    "from sklearn.pipeline import make_pipeline\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn_rvm import EMRVR\n",
    "\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "EPS = 1e-2\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_clf(clf, X, y, train_index, test_index, include_indices=False):\n",
    "    print(f'Training with {len(train_index)} samples; Testing with {len(test_index)} samples')\n",
    "    \n",
    "    # Get Train/Test Sets\n",
    "    X_train, X_test = X.iloc[train_index].values, X.iloc[test_index].values\n",
    "    y_train, y_test = y.iloc[train_index].values, y.iloc[test_index].values\n",
    "\n",
    "    # Train Model on Selected Fold\n",
    "    clf.fit(X_train, y_train)\n",
    "    y_pred = np.clip(clf.predict(X_test), 0, 1)\n",
    "    \n",
    "    # Organize and Return Results\n",
    "    result = dict(\n",
    "        instance = clf,\n",
    "        X_train=X_train,\n",
    "        X_test=X_test,\n",
    "        y_train = y_train,        \n",
    "        y_true = y_test,\n",
    "        y_pred = y_pred,\n",
    "        acc = clf.score(X_test, y_test),\n",
    "        mae = mean_absolute_error(y_test, y_pred),\n",
    "        rmse = mean_squared_error(y_test, y_pred, squared=False),\n",
    "        rmsle = mean_squared_log_error(y_test, y_pred, squared=False),\n",
    "    )\n",
    "        \n",
    "    if include_indices:\n",
    "        result.update({\n",
    "            'train_indices':train_index,\n",
    "            'test_indices':test_index\n",
    "        })\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading and Preparing CRI Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "gdf_vcra = pd.read_pickle('./data/norway-dataset/oslo_jan_mar_2019_4w_prep_encountering.vcra_dataset_v14.pickle')\n",
    "gdf_vcra.loc[:, 'ves_cri_bin'] = pd.cut(\n",
    "    gdf_vcra.ves_cri, \n",
    "    bins=np.arange(0, 1.1, .2),  \n",
    "    right=True, \n",
    "    include_lowest=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEtCAYAAADz1SBvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASD0lEQVR4nO3de4ymZXnH8e9PVkSpoBYaW2C7NKIBrS26sqaNFfEABBG1akAtpVLRGqxabVyNMaX2QLWppBZrSCRgPFCkVkG2gvGEWqDAagu4ASmuslJFlEOtqEWu/vG+6zsMe3g3c7jfuZ/vJ5lk5nlnZq9cDL955nru535SVUiS+vKg1gVIkhaf4S5JHTLcJalDhrskdchwl6QOGe6S1KFVrQsA2GeffWrNmjWty5CkFeWaa665var23dZrMxHua9as4eqrr25dhiStKEm+ub3Xmo5lkhyb5Ky77rqrZRmS1J2m4V5VF1XVKXvvvXfLMiSpO15QlaQOGe6S1CHDXZI6ZLhLUocMd0nqkEshJalDTW9iqqqLgIvWrl37ypZ1SBqWNesvbl0Cm08/Zkm/v2MZSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkNsPSFKHfBKTJHXIsYwkdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHFj3ckxye5ItJ3pfk8MX+/pKknZsq3JOcneS2JNfNO35UkhuS3JRk/fhwAT8E9gC2LG65kqRpTHvmfg5w1NwDSXYDzgSOBg4BTkhyCPDFqjoaeDNw2uKVKkma1lThXlWXAT+Yd/gw4KaqurmqfgqcBxxXVfeNX78DeMiiVSpJmtqqBXztfsAtcz7eAqxL8kLgSOARwD9s74uTnAKcArB69eoFlCFJmm8h4Z5tHKuq+hjwsZ19cVWdBZwFsHbt2lpAHZKkeRayWmYLcMCcj/cHbt2Vb+ADsiVpaSwk3K8CDkpyYJLdgeOBC3flG/iAbElaGtMuhfwIcDnwuCRbkpxcVfcCpwKXAJuA86vq+qUrVZI0ralm7lV1wnaObwA2LGpFkqQFa7r9gDN3SVoaTcPdmbskLQ03DpOkDjmWkaQOOZaRpA45lpGkDhnuktQhZ+6S1CFn7pLUIccyktQhw12SOmS4S1KHvKAqSR3ygqokdWghj9mbKWvWX9y6BDaffkzrEiQJcOYuSV0y3CWpQ4a7JHXI1TKS1CFXy0hShxzLSFKHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA65zl2SOuQ6d0nqkGMZSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUoe8Q1WSOuQdqpLUIccyktQhw12SOmS4S1KHDHdJ6tCq1gVIWh5r1l/cugQ2n35M6xIGwzN3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUoeWJNyT7JnkmiTPXYrvL0nasanCPcnZSW5Lct2840cluSHJTUnWz3npzcD5i1moJGl60565nwMcNfdAkt2AM4GjgUOAE5IckuRZwNeA7y5inZKkXTDV3jJVdVmSNfMOHwbcVFU3AyQ5DzgO+AVgT0aBf0+SDVV13+KVLEnamYVsHLYfcMucj7cA66rqVIAkJwG3by/Yk5wCnAKwevXqBZQhSZpvIRdUs41j9fN3qs6pqk9u74ur6qyqWltVa/fdd98FlCFJmm8h4b4FOGDOx/sDty6sHEnSYlhIuF8FHJTkwCS7A8cDF+7KN0hybJKz7rrrrgWUIUmab9qlkB8BLgcel2RLkpOr6l7gVOASYBNwflVdvyv/eFVdVFWn7L333rtatyRpB6ZdLXPCdo5vADYsakWSpAVruv2AYxlJWhpNw92xjCQtDTcOk6QOOZaRpA45lpGkDjmWkaQOGe6S1CFn7pLUIWfuktQhxzKS1CHDXZI6ZLhLUoe8oCpJHfKCqiR1yLGMJHXIcJekDhnuktQhw12SOuRqGUnqkKtlJKlDjmUkqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtSh1znLkkdcp27JHXIsYwkdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXI7QckqUNuPyBJHXIsI0kdMtwlqUOrWhcgLaU16y9uXQKbTz+mdQkaIM/cJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4se7kkOTvK+JBck+aPF/v6SpJ2bKtyTnJ3ktiTXzTt+VJIbktyUZD1AVW2qqlcDLwHWLn7JkqSdmfbM/RzgqLkHkuwGnAkcDRwCnJDkkPFrzwO+BHxm0SqVJE1tqnCvqsuAH8w7fBhwU1XdXFU/Bc4Djht//oVV9VvAyxazWEnSdBayK+R+wC1zPt4CrEtyOPBC4CHAhu19cZJTgFMAVq9evYAyJEnzLSTcs41jVVWfBz6/sy+uqrOAswDWrl1bC6hDkjTPQlbLbAEOmPPx/sCtCytHkrQYFhLuVwEHJTkwye7A8cCFu/INfEC2JC2NaZdCfgS4HHhcki1JTq6qe4FTgUuATcD5VXX9rvzjPiBbkpbGVDP3qjphO8c3sIOLppKkNppuP+BYRpKWRtNwdywjSUvDjcMkqUOOZSSpQ45lJKlDjmUkqUOGuyR1yHCXpA55QVWSOuQFVUnqkGMZSeqQ4S5JHXLmLkkdcuYuSR1yLCNJHTLcJalDhrskdchwl6QOuVpGkjrkahlJ6pBjGUnqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQh17lLUodc5y5JHXIsI0kdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHXL7AUnqkNsPSFKHHMtIUocMd0nq0KrWBWjxrVl/cesS2Hz6Ma1LkAbNM3dJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtSh1JVrWsgyfeAb7auA9gHuL11ETPAPkzYiwl7MTErvfjVqtp3Wy/MRLjPiiRXV9Xa1nW0Zh8m7MWEvZhYCb1wLCNJHTLcJalDhvv9ndW6gBlhHybsxYS9mJj5Xjhzl6QOeeYuSR0y3CWpQ4a7JHVokE9iSvL3U3za3VX1tiUvprEkT5ri0/6vqq5d8mIasxcT9mJipfZikBdUk3wTePtOPm19VR28HPW0lOR/gKuA7ODTDqyqNctTUTv2YsJeTKzUXgzyzB14d1Wdu6NPSPLI5Sqmsauq6ogdfUKSzy5XMY3Ziwl7MbEiezHIM3dJ6t1gL6gmOTLJyUnWzDv+ikYlzYwkr2ldQwtJdk+SOR8/I8kbkxzdsq5WkqxO8ojx+2uSvCjJExqX1URG1iV5YZIXjN/f0ZimuUGeuSf5a+C3gY3AscAZVfWe8Wsbq2qaCyhdSPIn8w8BbwH+CqCq/m7Zi2okyX8Ah1fVHUn+FHgBsAF4OnB1Vb2laYHLKMl64FXAT4C/Bd4EfBl4KvD+gf1cPAd4L/B14Nvjw/sDjwFeU1WXtqptR4Ya7tcCh1bVveMzkw8DN1TVG5J8paoObVvh8hlfLNoAXM/kgtHrgTMAquq0JoU1kOS6qnrC+P2rgadV1T1JVgEbq+qJbStcPkmuB9YCDwM2A79WVd9Lsidw5dY+DUGSTcDRVbV53vEDgQ2zuvBiqGOZVVV1L0BV3cno7H2vJB8Fdm9ZWAOPB3YD9gTeNQ7zO6rqtCEF+9jdc8YOtwN7jN9fxfD+X/lZVd0D3AncA3wfoKr+t2VRjawCtmzj+LeBBy9zLVMb6mqZ/0ry9Kr6AkBV/Qw4OclfAL/btrTlVVXfAl6U5Djg00ne3bqmhl4NfGg8nrkNuDrJF4AnMh5TDcjGJB9m9Ev/M8C5ST4FHAF8rWlly+9s4Kok5wG3jI8dABwPvL9ZVTsx1LHMQwHGZybzX9uvqr79wK/qX5KHAacB66rqd1rX00KS3YDnAI9lcsZ2yfgvvMEYj6JeDBRwAXAY8FLgW8CZQzuDT3IwcBywH6Px5Rbgwqqa2V90gwx3Serd0OaIO5VkY+saZkWSP2tdw6ywFxP2YmKWe2G4zzOkZZBTuKZ1ATPEXkzYi4mZ7cXgxzJJHgVUVd3RuhZJWiyDDPckq4F3As9ktNQrwF7AZxltGLa5WXENJDkSeD6ji0UF3Ap8oqo+1bKuFuzFhL3YuSRvr6o/b13Htgw13C9ndJPOBeNlkFtXSbwYeH1VPbVhecsqyRmMVoZ8gMla3v2BE4GvV9XrGpW27OzFhL2YTpJvVdXq1nVsy1DD/etVddCuvtajJDdW1WO3cTzAjfbCXsw7PsRe3L29l4CHVtVM3i801Auq1yR573jzn18Zv61L8l7gK62LW2Y/TnLYNo4/BfjxchfTmL2YsBcTdwIHVdVe894eDvx349q2ayZ/4yyDE4GTGd2ws/WmhFuAi5jhO86WyEnAPyZ5OJM/vw8A7h6/NiQnYS+2Ogl7sdUHgF8FvruN1z68zLVMbZBjGT1Qkkcz5+67qvpO45KasRcT9mLlMtwlqUNDnblLUtcMd0nqkOE+R5LjkqxrXYckLZThfn/rgLcl+dfWhbSWZNP47dTWtbRmLybsxcSs92KoSyG3qare2rqGWVFVByfZh9EvvEEb9+IXGT0/dNDsxcSs98LVMvMkeXZVfbp1HZoNSZ5UVYPfBjrJXsBBwM1usgdJ9qmq21vXsSOOZR5oUDcxJTkgyXlJvpjkrUkePOe1jzcsbdkledK8tycDFyY5NMmgtoJO8sHxX25bNxC7Hvgb4KtJXty0uGWW5Ogk30jypfHPwvXAlUm2JHlm6/q2Z5Bn7kku3N5LwBFVtedy1tNSkk8D/wxcweiu3ScDx1bV95N8paoObVrgMkpyH6M+/GTO4aeOj1VVHdGksAaSXFtVvz5+/9+Al1bV5nHgf6aqfqNthcsnyVeBE4BHAJ8EjqmqK8aP3vvQrD4DYqgz96cBLwd+OO94GD0rckj2rar3jd9/bZKXA5cleR6jbV6H5CXAa4F3VdUGgCTfqKpntC2riQcl2auq7gbuY/TsVKrq9vHzVYfkvqraBJDkR1V1BUBVbUoys9OPof1H2uoK4EdV9YX5LyS5oUE9LT04yR5V9WOAqvpgku8AlzB68v1gVNUFST4FvCPJHwBvZHi/4LY6DfhckjOBLwMfTfIJ4AhgaPu535nkVYye+XBHkjcA5wPP4oEniDNjkGMZTYx/UDfO/0WX5FDgnVX17DaVtZXkN4F3A4+vql9qXE4TSR4DvJLRvu6rGG0g9vGquqRpYcssyQHA2xj9BXMaoxHNycA3gTdtPaufNYMPdx+zp+0Z713+8PFoQlpRZnZetJSSrB6vEPkecCVwVZLbxsfWNC5vZiR5busaWqqRu8FezGUvJma5F4MMd+CfgH8BHl1VB1XVY4BfBj4OnNeysBnzlNYFzBB7MWEvJma2F4Mcy/iYPUm9G+pqmWvGj9Q7l9ETmGD0lJnfZ3iP2SPJ3sBR3P8p95dU1Z0t62rBXkzYi4mV2IuhjmVOBK5ldOX7EuDS8fvXAb/XsK5ll+REYCNwOPAwRssfn8HoF+CJDUtbdvZiwl5MrNReDHIso4nxuv51889AkjwSuLKqHtuksAbsxYS9mFipvRjqWGbrfhnP5/5/Zn2iqoZ2g0bY9o06941fGxJ7MWEvJlZkLwYZ7knOYHRjxgeYPNl9f+CPkxxdVa9rVVsDfwlsTHIpk+sPq4FnA+9oVlUb9mLCXkysyF4MciyT5MZt/Sk1vmnlxqGtlhn/eXkkc55yz+hi0eBu7LIXE/ZiYiX2Yqjh/p/AH1bVv887fhjw/q274Q1BktROfgim+Zwe2IsJezGxUnsx1NUyJwHvSfK1JJeO3zYB7xm/NiSfS/LaJKvnHkyye5IjkpzLaInoENiLCXsxsSJ7Mcgz962SPJo5f2ZV1Xcal7TskuwBvAJ4GXAgcCfwUEa/+C8Fzqyqr7aqbznZiwl7MbFSezHocNf9ZfQUpn2Ae2b55ozlYC8m7MXESuqF4T5Pko2z+mQVSZqW4S5JHRrqBdWfS/Ko8TInSerGIMPd/dwl9W6Q4Y77uUvq3CBn7u7nLql3g9xbBvdzl9S5oZ65787o6eXHcf+9Ii5ktP3ATxqWJ0kLNshwl6TeDfWC6gMk2di6BklaLIb7xMxuui9Ju8pwn7i4dQGStFgGOXNfqfszS9K0hnrmviL3Z5akaQ31zH1F7s8sSdMaZLjPtZL2Z5akaQ0+3CWpR0OduUtS1wx3SeqQ4S5JHTLcJalDhrskdej/AZZ+ue1Fv07GAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = gdf_vcra.ves_cri_bin.value_counts(sort=False).plot.bar()\n",
    "ax.set_yscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Get a Stratified Subset (to ensure a \"fair\" comparison)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X, y, y_bin = gdf_vcra.iloc[:, :-2], gdf_vcra.iloc[:, -2], gdf_vcra.iloc[:, -1].astype('str')\n",
    "X_sub, _, y_sub, _, y_bin_sub, _ = train_test_split(X, y, y_bin, train_size=0.35, random_state=10, stratify=y_bin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# SVM-VCRA (Gang et al.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# svm_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course', 'target_course', 'relative_bearing_target_to_own']\n",
    "svm_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course_rad', 'target_course_rad', 'relative_bearing_target_to_own']\n",
    "\n",
    "svm_vcra_training_data = X_sub.loc[:, svm_vcra_features].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)\n",
    "clf = make_pipeline(StandardScaler(), SVR(gamma='auto', kernel='rbf', verbose=True))\n",
    "    \n",
    "svm_vcra_skf_results = Parallel(n_jobs=-1)(delayed(evaluate_clf)(\n",
    "    clf, svm_vcra_training_data, y_sub, train_index, test_index\n",
    ") for (train_index, test_index) in tqdm(skf.split(X_sub, y_bin_sub), total=skf.get_n_splits(X_sub, y_bin_sub)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_vcra_skf_results_df = pd.DataFrame(svm_vcra_skf_results)\n",
    "svm_vcra_skf_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_vcra_skf_results_df.to_pickle('./data/pickle/svm_vcra_skf_results_v14.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# RVM-VCRA (Park et al.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# rvm_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course', 'target_course', 'relative_bearing_target_to_own', 'own_length', 'target_length']\n",
    "rvm_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course_rad', 'target_course_rad', 'relative_bearing_target_to_own', 'own_length_nmi', 'target_length_nmi']\n",
    "\n",
    "rvm_vcra_training_data = X_sub.loc[:, rvm_vcra_features].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)\n",
    "clf = make_pipeline(StandardScaler(), EMRVR(kernel='rbf', verbose=True))\n",
    "    \n",
    "rvm_vcra_skf_results = Parallel(n_jobs=1)(delayed(evaluate_clf)(\n",
    "    clf, rvm_vcra_training_data, y_sub, train_index, test_index\n",
    ") for (train_index, test_index) in tqdm(skf.split(X_sub, y_bin_sub), total=skf.get_n_splits(X_sub, y_bin_sub)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rvm_vcra_skf_results_df = pd.DataFrame(rvm_vcra_skf_results)\n",
    "rvm_vcra_skf_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rvm_vcra_skf_results_df.to_pickle('./data/pickle/rvm_vcra_skf_results_v14.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# CART-VCRA (Li et al.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cart_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course', 'target_course', 'azimuth_angle_target_to_own']\n",
    "cart_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course_rad', 'target_course_rad', 'azimuth_angle_target_to_own']\n",
    "\n",
    "cart_vcra_training_data = X_sub.loc[:, cart_vcra_features].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)\n",
    "clf = make_pipeline(StandardScaler(), RandomForestRegressor(random_state=10, verbose=True))\n",
    "    \n",
    "cart_vcra_skf_results = Parallel(n_jobs=-1)(delayed(evaluate_clf)(\n",
    "    clf, cart_vcra_training_data, y_sub, train_index, test_index\n",
    ") for (train_index, test_index) in tqdm(skf.split(X_sub, y_bin_sub), total=skf.get_n_splits(X_sub, y_bin_sub)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cart_vcra_skf_results_df = pd.DataFrame(cart_vcra_skf_results)\n",
    "cart_vcra_skf_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cart_vcra_skf_results_df.to_pickle('./data/pickle/cart_vcra_skf_results_v14.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "# MLP-VCRA (Ours)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # mlp_vcra_features = ['own_speed', 'own_course', 'target_speed', 'target_course', 'dist_euclid', 'relative_bearing_target_to_own', 'rel_movement_direction']\n",
    "# mlp_vcra_features = ['own_speed', 'own_course', 'target_speed', 'target_course', 'dist_euclid', 'azimuth_angle_target_to_own', 'rel_movement_direction']\n",
    "mlp_vcra_features = ['own_speed', 'own_course_rad', 'target_speed', 'target_course_rad', 'dist_euclid', 'azimuth_angle_target_to_own', 'rel_movement_direction']\n",
    "\n",
    "mlp_vcra_training_data = X_sub.loc[:, mlp_vcra_features].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)\n",
    "regr = make_pipeline(\n",
    "    StandardScaler(), \n",
    "    MLPRegressor(random_state=10, max_iter=300, hidden_layer_sizes=(256, 32), \n",
    "                 verbose=True, early_stopping=True, n_iter_no_change=7)\n",
    ")\n",
    "\n",
    "mlp_vcra_skf_results = Parallel(n_jobs=-1)(delayed(evaluate_clf)(\n",
    "    regr, mlp_vcra_training_data, y_sub, train_index, test_index\n",
    ") for (train_index, test_index) in tqdm(skf.split(X_sub, y_bin_sub), total=skf.get_n_splits(X_sub, y_bin_sub)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_vcra_skf_results_df = pd.DataFrame(mlp_vcra_skf_results)\n",
    "mlp_vcra_skf_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "mlp_vcra_skf_results_df.to_pickle('./data/pickle/mlp_vcra_skf_results_v14.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Combining Results to a Unified DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cart_vcra_skf_results_df = pd.read_pickle('./data/pickle/cart_vcra_skf_results_v14.pickle')\n",
    "svm_vcra_skf_results_df = pd.read_pickle('./data/pickle/svm_vcra_skf_results_v14.pickle')\n",
    "rvm_vcra_skf_results_df = pd.read_pickle('./data/pickle/rvm_vcra_skf_results_v14.pickle')\n",
    "mlp_vcra_skf_results_df = pd.read_pickle('./data/pickle/mlp_vcra_skf_results_v14.pickle')\n",
    "\n",
    "skf_results_df = pd.concat({\n",
    "    'cart_vcra': cart_vcra_skf_results_df, \n",
    "    'svm_vcra': svm_vcra_skf_results_df,\n",
    "    'rvm_vcra': rvm_vcra_skf_results_df,\n",
    "    'mlp_vcra': mlp_vcra_skf_results_df\n",
    "}, names=['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf_results_df.loc[:, 'r2_score'] = skf_results_df.apply(lambda l: sklearn.metrics.r2_score(l.y_true, l.y_pred), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>mae</th>\n",
       "      <th>rmse</th>\n",
       "      <th>rmsle</th>\n",
       "      <th>r2_score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cart_vcra</th>\n",
       "      <td>0.851717</td>\n",
       "      <td>0.039487</td>\n",
       "      <td>0.116528</td>\n",
       "      <td>0.085264</td>\n",
       "      <td>0.851717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>svm_vcra</th>\n",
       "      <td>0.568810</td>\n",
       "      <td>0.119438</td>\n",
       "      <td>0.196918</td>\n",
       "      <td>0.145188</td>\n",
       "      <td>0.576567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rvm_vcra</th>\n",
       "      <td>0.572970</td>\n",
       "      <td>0.127159</td>\n",
       "      <td>0.193644</td>\n",
       "      <td>0.137909</td>\n",
       "      <td>0.590550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp_vcra</th>\n",
       "      <td>0.959264</td>\n",
       "      <td>0.024619</td>\n",
       "      <td>0.060669</td>\n",
       "      <td>0.043949</td>\n",
       "      <td>0.959734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                acc       mae      rmse     rmsle  r2_score\n",
       "model                                                      \n",
       "cart_vcra  0.851717  0.039487  0.116528  0.085264  0.851717\n",
       "svm_vcra   0.568810  0.119438  0.196918  0.145188  0.576567\n",
       "rvm_vcra   0.572970  0.127159  0.193644  0.137909  0.590550\n",
       "mlp_vcra   0.959264  0.024619  0.060669  0.043949  0.959734"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skf_results_df.groupby(level=0, sort=False).mean(numeric_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf_results_df.to_pickle('./data/pickle/vcra_skf_results_df_v14.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Li et. al vs. VCRA/F @ Full Dataset\n",
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Split to Train/Test set (to ensure a \"fair\" comparison)\n",
    "X, y, y_bin = gdf_vcra.iloc[:, :-2], gdf_vcra.iloc[:, -2], gdf_vcra.iloc[:, -1].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Li et al."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cart_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course', 'target_course', 'azimuth_angle_target_to_own']\n",
    "cart_vcra_features = ['dist_euclid', 'own_speed', 'target_speed', 'own_course_rad', 'target_course_rad', 'azimuth_angle_target_to_own']\n",
    "\n",
    "cart_vcra_training_data = X.loc[:, cart_vcra_features].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)\n",
    "cart_vcra_lietal = make_pipeline(StandardScaler(), RandomForestRegressor(random_state=10, verbose=True))\n",
    "    \n",
    "cart_vcra_skf_results = Parallel(n_jobs=-1)(delayed((evaluate_clf))(\n",
    "    cart_vcra_lietal, cart_vcra_training_data, y, train_index, test_index, include_indices=True\n",
    ") for (train_index, test_index) in tqdm(skf.split(X, y_bin), total=skf.get_n_splits(X, y_bin)))\n",
    "\n",
    "cart_vcra_skf_results_df = pd.DataFrame(cart_vcra_skf_results)\n",
    "cart_vcra_skf_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "cart_vcra_skf_results_df.to_pickle('./data/pickle/cart_vcra_skf_results_v14.trained_on_all_data.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VCRA/F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## mlp_vcra_features = ['own_speed', 'own_course', 'target_speed', 'target_course', 'dist_euclid', 'relative_bearing_target_to_own', 'rel_movement_direction']\n",
    "# mlp_vcra_features = ['own_speed', 'own_course', 'target_speed', 'target_course', 'dist_euclid', 'azimuth_angle_target_to_own', 'rel_movement_direction']\n",
    "mlp_vcra_features = ['own_speed', 'own_course_rad', 'target_speed', 'target_course_rad', 'dist_euclid', 'azimuth_angle_target_to_own', 'rel_movement_direction']\n",
    "\n",
    "mlp_vcra_training_data = X.loc[:, mlp_vcra_features].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=10)\n",
    "mlp_vcra_ours = make_pipeline(\n",
    "    StandardScaler(), \n",
    "    MLPRegressor(random_state=10, max_iter=100, hidden_layer_sizes=(256, 32), \n",
    "                 verbose=True, early_stopping=True, n_iter_no_change=10)\n",
    ")\n",
    "\n",
    "mlp_vcra_skf_results = Parallel(n_jobs=-1)(delayed(evaluate_clf)(\n",
    "    mlp_vcra_ours, mlp_vcra_training_data, y, train_index, test_index, include_indices=True\n",
    ") for (train_index, test_index) in tqdm(skf.split(X, y_bin), total=skf.get_n_splits(X, y_bin)))\n",
    "\n",
    "mlp_vcra_skf_results_df = pd.DataFrame(mlp_vcra_skf_results)\n",
    "mlp_vcra_skf_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "mlp_vcra_skf_results_df.to_pickle('./data/pickle/mlp_vcra_skf_results_v14.trained_on_all_data.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combining Results to a Unified DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cart_vcra_skf_results_df = pd.read_pickle('./data/pickle/cart_vcra_skf_results_v14.trained_on_all_data.pickle')\n",
    "mlp_vcra_skf_results_df = pd.read_pickle('./data/pickle/mlp_vcra_skf_results_v14.trained_on_all_data.pickle')\n",
    "\n",
    "skf_results_df = pd.concat({\n",
    "    'cart_vcra': cart_vcra_skf_results_df, \n",
    "    'mlp_vcra': mlp_vcra_skf_results_df\n",
    "}, names=['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "skf_results_df.loc[:, 'r2_score'] = skf_results_df.apply(lambda l: sklearn.metrics.r2_score(l.y_true, l.y_pred), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "skf_results_df.to_pickle('./data/pickle/vcra_skf_results_df.cart_vs_mlp.trained_on_all_data.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>mae</th>\n",
       "      <th>rmse</th>\n",
       "      <th>rmsle</th>\n",
       "      <th>r2_score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cart_vcra</th>\n",
       "      <td>0.886719</td>\n",
       "      <td>0.031447</td>\n",
       "      <td>0.101848</td>\n",
       "      <td>0.074772</td>\n",
       "      <td>0.886719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp_vcra</th>\n",
       "      <td>0.969927</td>\n",
       "      <td>0.019279</td>\n",
       "      <td>0.052141</td>\n",
       "      <td>0.037807</td>\n",
       "      <td>0.970156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                acc       mae      rmse     rmsle  r2_score\n",
       "model                                                      \n",
       "cart_vcra  0.886719  0.031447  0.101848  0.074772  0.886719\n",
       "mlp_vcra   0.969927  0.019279  0.052141  0.037807  0.970156"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skf_results_df.groupby(level=0).mean(numeric_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>(-0.001, 0.2]</th>\n",
       "      <th>(0.2, 0.4]</th>\n",
       "      <th>(0.4, 0.6]</th>\n",
       "      <th>(0.6, 0.8]</th>\n",
       "      <th>(0.8, 1.0]</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cart_vcra</th>\n",
       "      <td>0.179458</td>\n",
       "      <td>0.066316</td>\n",
       "      <td>0.067501</td>\n",
       "      <td>0.058542</td>\n",
       "      <td>0.038912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp_vcra</th>\n",
       "      <td>0.086902</td>\n",
       "      <td>0.075979</td>\n",
       "      <td>0.049615</td>\n",
       "      <td>0.031168</td>\n",
       "      <td>0.021532</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           (-0.001, 0.2]  (0.2, 0.4]  (0.4, 0.6]  (0.6, 0.8]  (0.8, 1.0]\n",
       "model                                                                   \n",
       "cart_vcra       0.179458    0.066316    0.067501    0.058542    0.038912\n",
       "mlp_vcra        0.086902    0.075979    0.049615    0.031168    0.021532"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def metrics_per_cri_pct(sdf):\n",
    "    buckets = pd.cut(sdf.y_true, bins, right=True, include_lowest=True)\n",
    "    return pd.DataFrame({'y_true': sdf.y_true, 'y_pred': sdf.y_pred}).groupby(buckets).apply(\n",
    "        lambda l: mean_squared_log_error(l.y_true, l.y_pred, squared=False)\n",
    "    )\n",
    "    \n",
    "\n",
    "skf_results_df.apply(\n",
    "    lambda l: metrics_per_cri_pct(l), \n",
    "    axis=1\n",
    ").groupby(level=0).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "Out[71].columns = ['[0, 0.2]', '(0.2, 0.4]', '(0.4, 0.6]', '(0.6, 0.8]', '(0.8, 1.0]']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>[0, 0.2]</th>\n",
       "      <th>(0.2, 0.4]</th>\n",
       "      <th>(0.4, 0.6]</th>\n",
       "      <th>(0.6, 0.8]</th>\n",
       "      <th>(0.8, 1.0]</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cart_vcra</th>\n",
       "      <td>0.179458</td>\n",
       "      <td>0.066316</td>\n",
       "      <td>0.067501</td>\n",
       "      <td>0.058542</td>\n",
       "      <td>0.038912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp_vcra</th>\n",
       "      <td>0.086902</td>\n",
       "      <td>0.075979</td>\n",
       "      <td>0.049615</td>\n",
       "      <td>0.031168</td>\n",
       "      <td>0.021532</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           [0, 0.2]  (0.2, 0.4]  (0.4, 0.6]  (0.6, 0.8]  (0.8, 1.0]\n",
       "model                                                              \n",
       "cart_vcra  0.179458    0.066316    0.067501    0.058542    0.038912\n",
       "mlp_vcra   0.086902    0.075979    0.049615    0.031168    0.021532"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Out[71]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "26a0fb21edfd8898ccaa2d3be90ad05e47a6ebc73217c056c6f09f953e946973"
  },
  "kernelspec": {
   "display_name": "Python (VesselAI)",
   "language": "python",
   "name": "vesai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}