experiments/evaluation/evaluation_per_algorithm/TCN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from biopsykit.utils.dataframe_handling import multi_xs\n",
    "import optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fau_colors import register_cmaps\n",
    "register_cmaps()\n",
    "\n",
    "sns.set_theme(context=\"talk\", style=\"white\", palette=\"faculties\", font_scale=1.2)\n",
    "\n",
    "%matplotlib widget"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def conf_matrix(confusion_matrix):\n",
    "    x = confusion_matrix.sum(axis=1)[:, np.newaxis]\n",
    "    confusion_matrix /=x\n",
    "    confusion_matrix *= 100\n",
    "    confusion_matrix = round(confusion_matrix,2)\n",
    "    return confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### Fill in Algorithm here!! ####\n",
    "algorithm = \"TCN\"\n",
    "##### Fill in Algorithm here!! ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path.cwd().parents[2].joinpath(\"exports/results_per_algorithm/\")\n",
    "path = path.joinpath(algorithm)\n",
    "# path where database files are stored\n",
    "db_path = path.joinpath(\"db\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 5 Stage\n",
    "# Acc only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"5stage\"\n",
    "mod = \"acc\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.559608</td>\n",
       "      <td>0.099682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.470847</td>\n",
       "      <td>0.101369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>24.267857</td>\n",
       "      <td>39.094569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.292260</td>\n",
       "      <td>0.130258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.350942</td>\n",
       "      <td>0.134124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>746.267857</td>\n",
       "      <td>187.384882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.449138</td>\n",
       "      <td>0.105496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.559608</td>\n",
       "      <td>0.099682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>76.085193</td>\n",
       "      <td>15.863919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>137.334821</td>\n",
       "      <td>93.160643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>70.000000</td>\n",
       "      <td>79.082406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.848509</td>\n",
       "      <td>0.070758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>141.540179</td>\n",
       "      <td>117.377541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1025.352679</td>\n",
       "      <td>151.069953</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.559608    0.099682\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.470847    0.101369\n",
       "getup_latency             24.267857   39.094569\n",
       "kappa                      0.292260    0.130258\n",
       "mcc                        0.350942    0.134124\n",
       "net_sleep_duration       746.267857  187.384882\n",
       "precision                  0.449138    0.105496\n",
       "recall                     0.559608    0.099682\n",
       "sleep_efficiency          76.085193   15.863919\n",
       "sleep_onset              137.334821   93.160643\n",
       "sleep_onset_latency       70.000000   79.082406\n",
       "specificity                0.848509    0.070758\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   141.540179  117.377541\n",
       "wake_onset              1025.352679  151.069953"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>n1</th>\n",
       "      <th>n2</th>\n",
       "      <th>n3</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>55864.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24993.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n1</th>\n",
       "      <td>5978.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17123.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n2</th>\n",
       "      <td>10920.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>82159.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n3</th>\n",
       "      <td>846.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16238.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>2184.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29625.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake   n1       n2   n3   rem\n",
       "wake  55864.0  0.0  24993.0  0.0   2.0\n",
       "n1     5978.0  0.0  17123.0  0.0   1.0\n",
       "n2    10920.0  0.0  82159.0  0.0   4.0\n",
       "n3      846.0  0.0  16238.0  0.0   0.0\n",
       "rem    2184.0  0.0  29625.0  0.0  11.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 5 Stage\n",
    "# Acc + HRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"5stage\"\n",
    "mod = \"acc_hrv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.607875</td>\n",
       "      <td>0.094779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.556259</td>\n",
       "      <td>0.098679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>8.803571</td>\n",
       "      <td>22.834102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.399357</td>\n",
       "      <td>0.126586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.431368</td>\n",
       "      <td>0.124306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>752.187500</td>\n",
       "      <td>164.740101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.595690</td>\n",
       "      <td>0.102672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.607875</td>\n",
       "      <td>0.094779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>76.687279</td>\n",
       "      <td>12.927083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>110.183036</td>\n",
       "      <td>84.210645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>42.848214</td>\n",
       "      <td>57.576809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.855752</td>\n",
       "      <td>0.064064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>164.397321</td>\n",
       "      <td>113.007870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1040.816964</td>\n",
       "      <td>148.113420</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.607875    0.094779\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.556259    0.098679\n",
       "getup_latency              8.803571   22.834102\n",
       "kappa                      0.399357    0.126586\n",
       "mcc                        0.431368    0.124306\n",
       "net_sleep_duration       752.187500  164.740101\n",
       "precision                  0.595690    0.102672\n",
       "recall                     0.607875    0.094779\n",
       "sleep_efficiency          76.687279   12.927083\n",
       "sleep_onset              110.183036   84.210645\n",
       "sleep_onset_latency       42.848214   57.576809\n",
       "specificity                0.855752    0.064064\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   164.397321  113.007870\n",
       "wake_onset              1040.816964  148.113420"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>n1</th>\n",
       "      <th>n2</th>\n",
       "      <th>n3</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>57334.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>19011.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>4280.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n1</th>\n",
       "      <td>4489.0</td>\n",
       "      <td>171.0</td>\n",
       "      <td>16168.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>2233.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n2</th>\n",
       "      <td>7909.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>78818.0</td>\n",
       "      <td>1204.0</td>\n",
       "      <td>4967.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n3</th>\n",
       "      <td>766.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14837.0</td>\n",
       "      <td>866.0</td>\n",
       "      <td>613.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>3400.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>15530.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>12798.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake     n1       n2      n3      rem\n",
       "wake  57334.0  187.0  19011.0    47.0   4280.0\n",
       "n1     4489.0  171.0  16168.0    41.0   2233.0\n",
       "n2     7909.0  185.0  78818.0  1204.0   4967.0\n",
       "n3      766.0    2.0  14837.0   866.0    613.0\n",
       "rem    3400.0   55.0  15530.0    37.0  12798.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 5 Stage\n",
    "# ACC + HRV + RRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"5stage\"\n",
    "mod = \"acc_hrv_RRV\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.614608</td>\n",
       "      <td>0.096677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.566281</td>\n",
       "      <td>0.100533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>10.428571</td>\n",
       "      <td>23.615840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.413200</td>\n",
       "      <td>0.129512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.441417</td>\n",
       "      <td>0.128087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>737.151786</td>\n",
       "      <td>173.553843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.610914</td>\n",
       "      <td>0.101386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.614608</td>\n",
       "      <td>0.096677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>75.179075</td>\n",
       "      <td>14.151927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>115.138393</td>\n",
       "      <td>84.050719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>47.803571</td>\n",
       "      <td>59.879077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.857109</td>\n",
       "      <td>0.062888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>173.901786</td>\n",
       "      <td>125.772730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1039.191964</td>\n",
       "      <td>149.509583</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.614608    0.096677\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.566281    0.100533\n",
       "getup_latency             10.428571   23.615840\n",
       "kappa                      0.413200    0.129512\n",
       "mcc                        0.441417    0.128087\n",
       "net_sleep_duration       737.151786  173.553843\n",
       "precision                  0.610914    0.101386\n",
       "recall                     0.614608    0.096677\n",
       "sleep_efficiency          75.179075   14.151927\n",
       "sleep_onset              115.138393   84.050719\n",
       "sleep_onset_latency       47.803571   59.879077\n",
       "specificity                0.857109    0.062888\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   173.901786  125.772730\n",
       "wake_onset              1039.191964  149.509583"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>n1</th>\n",
       "      <th>n2</th>\n",
       "      <th>n3</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>59428.0</td>\n",
       "      <td>589.0</td>\n",
       "      <td>16167.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>4663.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n1</th>\n",
       "      <td>5613.0</td>\n",
       "      <td>507.0</td>\n",
       "      <td>14327.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2651.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n2</th>\n",
       "      <td>8801.0</td>\n",
       "      <td>483.0</td>\n",
       "      <td>76587.0</td>\n",
       "      <td>736.0</td>\n",
       "      <td>6476.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n3</th>\n",
       "      <td>688.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>14762.0</td>\n",
       "      <td>687.0</td>\n",
       "      <td>926.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>3035.0</td>\n",
       "      <td>203.0</td>\n",
       "      <td>14186.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>14384.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake     n1       n2     n3      rem\n",
       "wake  59428.0  589.0  16167.0   12.0   4663.0\n",
       "n1     5613.0  507.0  14327.0    4.0   2651.0\n",
       "n2     8801.0  483.0  76587.0  736.0   6476.0\n",
       "n3      688.0   21.0  14762.0  687.0    926.0\n",
       "rem    3035.0  203.0  14186.0   12.0  14384.0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv_rrv = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv_rrv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 5 Stage\n",
    "# ACC + HRV + EDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"5stage\"\n",
    "mod = \"acc_hrv_EDR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.614637</td>\n",
       "      <td>0.095767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.565840</td>\n",
       "      <td>0.100467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>10.428571</td>\n",
       "      <td>25.277904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.415260</td>\n",
       "      <td>0.129389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.443839</td>\n",
       "      <td>0.127803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>747.656250</td>\n",
       "      <td>166.537227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.610534</td>\n",
       "      <td>0.107739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.614637</td>\n",
       "      <td>0.095767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>76.269685</td>\n",
       "      <td>13.387739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>122.187500</td>\n",
       "      <td>84.970006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>54.852679</td>\n",
       "      <td>64.726750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.856719</td>\n",
       "      <td>0.062812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>157.924107</td>\n",
       "      <td>111.493966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1039.191964</td>\n",
       "      <td>149.335039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.614637    0.095767\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.565840    0.100467\n",
       "getup_latency             10.428571   25.277904\n",
       "kappa                      0.415260    0.129389\n",
       "mcc                        0.443839    0.127803\n",
       "net_sleep_duration       747.656250  166.537227\n",
       "precision                  0.610534    0.107739\n",
       "recall                     0.614637    0.095767\n",
       "sleep_efficiency          76.269685   13.387739\n",
       "sleep_onset              122.187500   84.970006\n",
       "sleep_onset_latency       54.852679   64.726750\n",
       "specificity                0.856719    0.062812\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   157.924107  111.493966\n",
       "wake_onset              1039.191964  149.335039"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>n1</th>\n",
       "      <th>n2</th>\n",
       "      <th>n3</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>58621.0</td>\n",
       "      <td>376.0</td>\n",
       "      <td>16184.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>5659.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n1</th>\n",
       "      <td>5330.0</td>\n",
       "      <td>537.0</td>\n",
       "      <td>14079.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3156.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n2</th>\n",
       "      <td>8108.0</td>\n",
       "      <td>238.0</td>\n",
       "      <td>76795.0</td>\n",
       "      <td>403.0</td>\n",
       "      <td>7539.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n3</th>\n",
       "      <td>653.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>15146.0</td>\n",
       "      <td>376.0</td>\n",
       "      <td>906.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>2514.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>13779.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>15370.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake     n1       n2     n3      rem\n",
       "wake  58621.0  376.0  16184.0   19.0   5659.0\n",
       "n1     5330.0  537.0  14079.0    0.0   3156.0\n",
       "n2     8108.0  238.0  76795.0  403.0   7539.0\n",
       "n3      653.0    3.0  15146.0  376.0    906.0\n",
       "rem    2514.0  147.0  13779.0   10.0  15370.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv_edr = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv_edr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.2,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.0013624065848891523,\n",
       " 'n_hid': 136,\n",
       " 'num_chanels': 5}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.1,\n",
       " 'kernel_size': 5,\n",
       " 'learning_rate': 0.00032097501473072644,\n",
       " 'n_hid': 72,\n",
       " 'num_chanels': 4}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.30000000000000004,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.0015439020788631241,\n",
       " 'n_hid': 296,\n",
       " 'num_chanels': 5}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.30000000000000004,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.0010453056809232213,\n",
       " 'n_hid': 72,\n",
       " 'num_chanels': 6}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_RRV_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_EDR_\" + stage + \".db\")[0].best_trial.params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3 Stage\n",
    "# Acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"3stage\"\n",
    "mod = \"acc\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.582304</td>\n",
       "      <td>0.075697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.527674</td>\n",
       "      <td>0.078131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>32.522321</td>\n",
       "      <td>20.764307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.164196</td>\n",
       "      <td>0.107672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.185254</td>\n",
       "      <td>0.116763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>794.549107</td>\n",
       "      <td>145.997350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.515050</td>\n",
       "      <td>0.071811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.582304</td>\n",
       "      <td>0.075697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>80.918022</td>\n",
       "      <td>8.844442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>69.406250</td>\n",
       "      <td>69.893321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>2.071429</td>\n",
       "      <td>5.724175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specifity</th>\n",
       "      <td>0.665383</td>\n",
       "      <td>0.089786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>155.924107</td>\n",
       "      <td>86.790395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1017.098214</td>\n",
       "      <td>148.855482</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.582304    0.075697\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.527674    0.078131\n",
       "getup_latency             32.522321   20.764307\n",
       "kappa                      0.164196    0.107672\n",
       "mcc                        0.185254    0.116763\n",
       "net_sleep_duration       794.549107  145.997350\n",
       "precision                  0.515050    0.071811\n",
       "recall                     0.582304    0.075697\n",
       "sleep_efficiency          80.918022    8.844442\n",
       "sleep_onset               69.406250   69.893321\n",
       "sleep_onset_latency        2.071429    5.724175\n",
       "specifity                  0.665383    0.089786\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   155.924107   86.790395\n",
       "wake_onset              1017.098214  148.855482"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>nrem</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>31706.0</td>\n",
       "      <td>49153.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nrem</th>\n",
       "      <td>21684.0</td>\n",
       "      <td>111585.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>5189.0</td>\n",
       "      <td>26631.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake      nrem  rem\n",
       "wake  31706.0   49153.0  0.0\n",
       "nrem  21684.0  111585.0  0.0\n",
       "rem    5189.0   26631.0  0.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3 Stage\n",
    "# Acc + HRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"3stage\"\n",
    "mod = \"acc_hrv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.738244</td>\n",
       "      <td>0.092963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.720012</td>\n",
       "      <td>0.100729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>8.156250</td>\n",
       "      <td>22.988546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.508603</td>\n",
       "      <td>0.161568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.534760</td>\n",
       "      <td>0.152774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>815.107143</td>\n",
       "      <td>153.791418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.759871</td>\n",
       "      <td>0.084902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.738244</td>\n",
       "      <td>0.092963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>83.128035</td>\n",
       "      <td>10.618794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>106.580357</td>\n",
       "      <td>81.802029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>39.245536</td>\n",
       "      <td>54.063138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.828361</td>\n",
       "      <td>0.085147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>110.245536</td>\n",
       "      <td>87.981793</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1041.464286</td>\n",
       "      <td>149.007535</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.738244    0.092963\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.720012    0.100729\n",
       "getup_latency              8.156250   22.988546\n",
       "kappa                      0.508603    0.161568\n",
       "mcc                        0.534760    0.152774\n",
       "net_sleep_duration       815.107143  153.791418\n",
       "precision                  0.759871    0.084902\n",
       "recall                     0.738244    0.092963\n",
       "sleep_efficiency          83.128035   10.618794\n",
       "sleep_onset              106.580357   81.802029\n",
       "sleep_onset_latency       39.245536   54.063138\n",
       "specificity                0.828361    0.085147\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   110.245536   87.981793\n",
       "wake_onset              1041.464286  149.007535"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>nrem</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>50315.0</td>\n",
       "      <td>26525.0</td>\n",
       "      <td>4019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nrem</th>\n",
       "      <td>7009.0</td>\n",
       "      <td>119755.0</td>\n",
       "      <td>6505.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>1193.0</td>\n",
       "      <td>18814.0</td>\n",
       "      <td>11813.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake      nrem      rem\n",
       "wake  50315.0   26525.0   4019.0\n",
       "nrem   7009.0  119755.0   6505.0\n",
       "rem    1193.0   18814.0  11813.0"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3 Stage\n",
    "# Acc + HRV + RRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"3stage\"\n",
    "mod = \"acc_hrv_RRV\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.721091</td>\n",
       "      <td>0.085159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.699383</td>\n",
       "      <td>0.092366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>9.125000</td>\n",
       "      <td>23.477926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.468873</td>\n",
       "      <td>0.148802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.493207</td>\n",
       "      <td>0.144114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>802.803571</td>\n",
       "      <td>160.482224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.730900</td>\n",
       "      <td>0.080798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.721091</td>\n",
       "      <td>0.085159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>81.859281</td>\n",
       "      <td>11.698019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>101.053571</td>\n",
       "      <td>78.373412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>33.718750</td>\n",
       "      <td>49.721434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.812326</td>\n",
       "      <td>0.084119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>123.607143</td>\n",
       "      <td>101.808125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1040.495536</td>\n",
       "      <td>149.334007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.721091    0.085159\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.699383    0.092366\n",
       "getup_latency              9.125000   23.477926\n",
       "kappa                      0.468873    0.148802\n",
       "mcc                        0.493207    0.144114\n",
       "net_sleep_duration       802.803571  160.482224\n",
       "precision                  0.730900    0.080798\n",
       "recall                     0.721091    0.085159\n",
       "sleep_efficiency          81.859281   11.698019\n",
       "sleep_onset              101.053571   78.373412\n",
       "sleep_onset_latency       33.718750   49.721434\n",
       "specificity                0.812326    0.084119\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   123.607143  101.808125\n",
       "wake_onset              1040.495536  149.334007"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>nrem</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>50002.0</td>\n",
       "      <td>27633.0</td>\n",
       "      <td>3224.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nrem</th>\n",
       "      <td>9733.0</td>\n",
       "      <td>118415.0</td>\n",
       "      <td>5121.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>2132.0</td>\n",
       "      <td>20426.0</td>\n",
       "      <td>9262.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake      nrem     rem\n",
       "wake  50002.0   27633.0  3224.0\n",
       "nrem   9733.0  118415.0  5121.0\n",
       "rem    2132.0   20426.0  9262.0"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv_rrv = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv_rrv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# 3 Stage\n",
    "# Acc + HRV + EDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"3stage\"\n",
    "mod = \"acc_hrv_EDR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.704396</td>\n",
       "      <td>0.086156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.668486</td>\n",
       "      <td>0.094516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>7.941964</td>\n",
       "      <td>20.434764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.419408</td>\n",
       "      <td>0.149737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.458308</td>\n",
       "      <td>0.144902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>825.816964</td>\n",
       "      <td>152.292047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.715291</td>\n",
       "      <td>0.082679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.704396</td>\n",
       "      <td>0.086156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>84.223307</td>\n",
       "      <td>10.426018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>100.281250</td>\n",
       "      <td>78.353038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>32.946429</td>\n",
       "      <td>52.178864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.815864</td>\n",
       "      <td>0.088502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>103.687500</td>\n",
       "      <td>88.347970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1041.678571</td>\n",
       "      <td>148.427918</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.704396    0.086156\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.668486    0.094516\n",
       "getup_latency              7.941964   20.434764\n",
       "kappa                      0.419408    0.149737\n",
       "mcc                        0.458308    0.144902\n",
       "net_sleep_duration       825.816964  152.292047\n",
       "precision                  0.715291    0.082679\n",
       "recall                     0.704396    0.086156\n",
       "sleep_efficiency          84.223307   10.426018\n",
       "sleep_onset              100.281250   78.353038\n",
       "sleep_onset_latency       32.946429   52.178864\n",
       "specificity                0.815864    0.088502\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   103.687500   88.347970\n",
       "wake_onset              1041.678571  148.427918"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>nrem</th>\n",
       "      <th>rem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>46199.0</td>\n",
       "      <td>32658.0</td>\n",
       "      <td>2002.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nrem</th>\n",
       "      <td>8188.0</td>\n",
       "      <td>122492.0</td>\n",
       "      <td>2589.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rem</th>\n",
       "      <td>1680.0</td>\n",
       "      <td>25262.0</td>\n",
       "      <td>4878.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         wake      nrem     rem\n",
       "wake  46199.0   32658.0  2002.0\n",
       "nrem   8188.0  122492.0  2589.0\n",
       "rem    1680.0   25262.0  4878.0"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv_edr = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv_edr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.30000000000000004,\n",
       " 'kernel_size': 3,\n",
       " 'learning_rate': 0.3981177980768058,\n",
       " 'n_hid': 264,\n",
       " 'num_chanels': 4}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.5,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.0013126211552047244,\n",
       " 'n_hid': 328,\n",
       " 'num_chanels': 5}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.4,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.0004520321088832826,\n",
       " 'n_hid': 456,\n",
       " 'num_chanels': 3}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 256,\n",
       " 'dropout': 0.2,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.0009521080310306135,\n",
       " 'n_hid': 104,\n",
       " 'num_chanels': 3}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_RRV_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_EDR_\" + stage + \".db\")[0].best_trial.params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Binary\n",
    "# Acc only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"binary\"\n",
    "mod = \"acc\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.821436</td>\n",
       "      <td>0.103902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.861540</td>\n",
       "      <td>0.100519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>20.308036</td>\n",
       "      <td>33.180424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.563055</td>\n",
       "      <td>0.215931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.582803</td>\n",
       "      <td>0.205806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>772.084821</td>\n",
       "      <td>177.427069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.845666</td>\n",
       "      <td>0.115260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.894216</td>\n",
       "      <td>0.120742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>78.727439</td>\n",
       "      <td>14.450188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>137.607143</td>\n",
       "      <td>87.979804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>70.272321</td>\n",
       "      <td>76.277414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.659150</td>\n",
       "      <td>0.207774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>116.866071</td>\n",
       "      <td>108.169606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1029.312500</td>\n",
       "      <td>150.294809</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.821436    0.103902\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.861540    0.100519\n",
       "getup_latency             20.308036   33.180424\n",
       "kappa                      0.563055    0.215931\n",
       "mcc                        0.582803    0.205806\n",
       "net_sleep_duration       772.084821  177.427069\n",
       "precision                  0.845666    0.115260\n",
       "recall                     0.894216    0.120742\n",
       "sleep_efficiency          78.727439   14.450188\n",
       "sleep_onset              137.607143   87.979804\n",
       "sleep_onset_latency       70.272321   76.277414\n",
       "specificity                0.659150    0.207774\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   116.866071  108.169606\n",
       "wake_onset              1029.312500  150.294809"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>53500.0</td>\n",
       "      <td>27359.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep</th>\n",
       "      <td>16317.0</td>\n",
       "      <td>148772.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          wake     sleep\n",
       "wake   53500.0   27359.0\n",
       "sleep  16317.0  148772.0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Binary\n",
    "# Acc + HRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"binary\"\n",
    "mod = \"acc_hrv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.827096</td>\n",
       "      <td>0.092478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.871500</td>\n",
       "      <td>0.083592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>9.258929</td>\n",
       "      <td>24.159316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.567974</td>\n",
       "      <td>0.193507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.591836</td>\n",
       "      <td>0.179817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>801.866071</td>\n",
       "      <td>159.804898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.833743</td>\n",
       "      <td>0.114829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.925432</td>\n",
       "      <td>0.081413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>81.706774</td>\n",
       "      <td>11.279480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>105.517857</td>\n",
       "      <td>82.217908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>38.183036</td>\n",
       "      <td>55.817186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.624551</td>\n",
       "      <td>0.197480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>122.370536</td>\n",
       "      <td>93.249707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1040.361607</td>\n",
       "      <td>148.808688</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.827096    0.092478\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.871500    0.083592\n",
       "getup_latency              9.258929   24.159316\n",
       "kappa                      0.567974    0.193507\n",
       "mcc                        0.591836    0.179817\n",
       "net_sleep_duration       801.866071  159.804898\n",
       "precision                  0.833743    0.114829\n",
       "recall                     0.925432    0.081413\n",
       "sleep_efficiency          81.706774   11.279480\n",
       "sleep_onset              105.517857   82.217908\n",
       "sleep_onset_latency       38.183036   55.817186\n",
       "specificity                0.624551    0.197480\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   122.370536   93.249707\n",
       "wake_onset              1040.361607  148.808688"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>50030.0</td>\n",
       "      <td>30829.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep</th>\n",
       "      <td>11584.0</td>\n",
       "      <td>153505.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          wake     sleep\n",
       "wake   50030.0   30829.0\n",
       "sleep  11584.0  153505.0"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Binary\n",
    "# Acc + HRV + RRV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"binary\"\n",
    "mod = \"acc_hrv_RRV\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.842264</td>\n",
       "      <td>0.084711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.878204</td>\n",
       "      <td>0.082196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>8.080357</td>\n",
       "      <td>20.816145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.616684</td>\n",
       "      <td>0.177841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.632768</td>\n",
       "      <td>0.167812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>763.937500</td>\n",
       "      <td>161.545272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.862087</td>\n",
       "      <td>0.107081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.906925</td>\n",
       "      <td>0.091566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>77.887790</td>\n",
       "      <td>12.262552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>109.522321</td>\n",
       "      <td>82.925270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>42.187500</td>\n",
       "      <td>56.975729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.707078</td>\n",
       "      <td>0.177658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>154.441964</td>\n",
       "      <td>108.352838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1041.540179</td>\n",
       "      <td>149.534459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.842264    0.084711\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.878204    0.082196\n",
       "getup_latency              8.080357   20.816145\n",
       "kappa                      0.616684    0.177841\n",
       "mcc                        0.632768    0.167812\n",
       "net_sleep_duration       763.937500  161.545272\n",
       "precision                  0.862087    0.107081\n",
       "recall                     0.906925    0.091566\n",
       "sleep_efficiency          77.887790   12.262552\n",
       "sleep_onset              109.522321   82.925270\n",
       "sleep_onset_latency       42.187500   56.975729\n",
       "specificity                0.707078    0.177658\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   154.441964  108.352838\n",
       "wake_onset              1041.540179  149.534459"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>56704.0</td>\n",
       "      <td>24155.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep</th>\n",
       "      <td>14480.0</td>\n",
       "      <td>150609.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          wake     sleep\n",
       "wake   56704.0   24155.0\n",
       "sleep  14480.0  150609.0"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv_rrv = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv_rrv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "# Binary\n",
    "# Acc + HRV + EDR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "stage = \"binary\"\n",
    "mod = \"acc_hrv_EDR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_results = pd.read_csv(path.joinpath(algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col=\"metric\")\n",
    "full_results.columns.name = \"subject\"\n",
    "results = full_results.drop(index=\"confusion_matrix\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(results.stack(\"subject\"), columns=[\"data\"]).astype(\"float\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>metric</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.834515</td>\n",
       "      <td>0.091251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_end</th>\n",
       "      <td>1049.620536</td>\n",
       "      <td>148.279048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bed_interval_start</th>\n",
       "      <td>67.334821</td>\n",
       "      <td>70.022350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.876352</td>\n",
       "      <td>0.081824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>getup_latency</th>\n",
       "      <td>8.866071</td>\n",
       "      <td>22.804687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kappa</th>\n",
       "      <td>0.590900</td>\n",
       "      <td>0.188401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mcc</th>\n",
       "      <td>0.613190</td>\n",
       "      <td>0.175710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>net_sleep_duration</th>\n",
       "      <td>795.781250</td>\n",
       "      <td>157.646285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.842109</td>\n",
       "      <td>0.112537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.925976</td>\n",
       "      <td>0.082730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_efficiency</th>\n",
       "      <td>81.151801</td>\n",
       "      <td>11.389287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset</th>\n",
       "      <td>109.669643</td>\n",
       "      <td>80.351067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep_onset_latency</th>\n",
       "      <td>42.334821</td>\n",
       "      <td>55.085416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>0.649891</td>\n",
       "      <td>0.191074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>total_sleep_duration</th>\n",
       "      <td>983.285714</td>\n",
       "      <td>152.957389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_after_sleep_onset</th>\n",
       "      <td>124.928571</td>\n",
       "      <td>97.057906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wake_onset</th>\n",
       "      <td>1040.754464</td>\n",
       "      <td>149.059646</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               data            \n",
       "                               mean         std\n",
       "metric                                         \n",
       "accuracy                   0.834515    0.091251\n",
       "bed_interval_end        1049.620536  148.279048\n",
       "bed_interval_start        67.334821   70.022350\n",
       "f1                         0.876352    0.081824\n",
       "getup_latency              8.866071   22.804687\n",
       "kappa                      0.590900    0.188401\n",
       "mcc                        0.613190    0.175710\n",
       "net_sleep_duration       795.781250  157.646285\n",
       "precision                  0.842109    0.112537\n",
       "recall                     0.925976    0.082730\n",
       "sleep_efficiency          81.151801   11.389287\n",
       "sleep_onset              109.669643   80.351067\n",
       "sleep_onset_latency       42.334821   55.085416\n",
       "specificity                0.649891    0.191074\n",
       "total_sleep_duration     983.285714  152.957389\n",
       "wake_after_sleep_onset   124.928571   97.057906\n",
       "wake_onset              1040.754464  149.059646"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.groupby(\"metric\").agg([\"mean\", \"std\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>wake</th>\n",
       "      <th>sleep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>wake</th>\n",
       "      <td>51880.0</td>\n",
       "      <td>28979.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sleep</th>\n",
       "      <td>11641.0</td>\n",
       "      <td>153448.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          wake     sleep\n",
       "wake   51880.0   28979.0\n",
       "sleep  11641.0  153448.0"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix_acc_hrv_edr = pd.read_csv(path.joinpath(\"confusion_matrix_\" + algorithm +\"_benchmark_\" + mod + \"_\" + stage + \".csv\"), index_col = \"Unnamed: 0\")\n",
    "confusion_matrix_acc_hrv_edr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'batch_size': 128,\n",
       " 'dropout': 0.1,\n",
       " 'kernel_size': 5,\n",
       " 'learning_rate': 0.0015142744120143696,\n",
       " 'n_hid': 136,\n",
       " 'num_chanels': 5}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 128,\n",
       " 'dropout': 0.1,\n",
       " 'kernel_size': 4,\n",
       " 'learning_rate': 0.023556094317490377,\n",
       " 'n_hid': 488,\n",
       " 'num_chanels': 5}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 128,\n",
       " 'dropout': 0.2,\n",
       " 'kernel_size': 5,\n",
       " 'learning_rate': 0.0015597868247307512,\n",
       " 'n_hid': 232,\n",
       " 'num_chanels': 4}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'batch_size': 64,\n",
       " 'dropout': 0.5,\n",
       " 'kernel_size': 5,\n",
       " 'learning_rate': 0.0016510059260365054,\n",
       " 'n_hid': 232,\n",
       " 'num_chanels': 5}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_RRV_\" + stage + \".db\")[0].best_trial.params)\n",
    "display(optuna.get_all_study_summaries(\"sqlite:///\" + str(db_path) + \"/\" + algorithm + \"_acc_hrv_EDR_\" + stage + \".db\")[0].best_trial.params)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sleep-analysis",
   "language": "python",
   "name": "sleep-analysis"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}