model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "from torchvision import transforms\n",
    "from torchvision.models import resnet18\n",
    "from torchvision.models.resnet import ResNet18_Weights\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, precision_score\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "import itertools\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "import os\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MIDmodel2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MIDmodel2, self).__init__()\n",
    "        self.pretrained = resnet18(weights = ResNet18_Weights.DEFAULT)\n",
    "        self.pretrained.fc = nn.Identity()\n",
    "        self.fc1 = nn.Linear(512, 4)\n",
    "\n",
    "    def get_embeddings(self, x):\n",
    "        x = self.pretrained(x)\n",
    "        return x\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pretrained(x)\n",
    "        x = self.fc1(x)\n",
    "        return x\n",
    "    \n",
    "class ImageDataset(Dataset):\n",
    "    def __init__(self, root_dir, patients_ids, patients_df, transform=None):\n",
    "        self.patients_df = patients_df[patients_df[\"patient_id\"].isin(patients_ids)]\n",
    "        self.root_dir = root_dir\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.patients_df)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        row = self.patients_df.iloc[idx]\n",
    "        path = os.path.join(self.root_dir, row[\"patient_id\"] + \"_\" + row[\"exam_id\"] + \"_\" + row[\"spot\"] + \"_\" + row[\"frame_number\"] + \"_\" + row[\"score\"] + \".png\")\n",
    "        image = Image.open(path)\n",
    "        label = row[\"score\"]\n",
    "\n",
    "        if self.transform:\n",
    "            image = self.transform(image)\n",
    "            \n",
    "        image = transforms.ToTensor()(image)\n",
    "        image = transforms.Resize((224, 224))(image)\n",
    "        image = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])(image)\n",
    "\n",
    "        return image, int(label), path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1048', '1069', '1017', '1051', '1052', '1067', '1047', '1050']\n",
      "score\n",
      "0    7888\n",
      "1    7540\n",
      "2    7189\n",
      "3    7592\n",
      "Name: patient_id, dtype: int64\n",
      "['1045', '1066', '1068']\n",
      "score\n",
      "0    6625\n",
      "1    1721\n",
      "2    5535\n",
      "3    3631\n",
      "Name: patient_id, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"images/\"\n",
    "images_paths = glob.glob(f\"{data_dir}*.png\", recursive=True)\n",
    "images_df = pd.DataFrame([path[len(data_dir):-4].split(\"_\") for path in images_paths], columns=[\"patient_id\", \"exam_id\", \"spot\", \"frame_number\", \"score\"])\n",
    "images_df[\"score\"] = images_df[\"score\"].astype(str)\n",
    "images_df[\"frame_number\"] = images_df[\"frame_number\"].astype(str)\n",
    "images_df[\"spot\"] = images_df[\"spot\"].astype(str)\n",
    "images_df[\"patient_id\"] = images_df[\"patient_id\"].astype(str)\n",
    "images_df[\"exam_id\"] = images_df[\"exam_id\"].astype(str)\n",
    "patients_ids = set(images_df[\"patient_id\"])\n",
    "patients_ids = list(patients_ids)\n",
    "\n",
    "# select the 8 patients for training based on the most balanced distribution of the scores\n",
    "combs = list(itertools.combinations(patients_ids, 8))\n",
    "stds = []\n",
    "for i, c in enumerate(combs):\n",
    "    stds.append((images_df[images_df[\"patient_id\"].isin(c)].groupby(\"score\").count()[\"patient_id\"].std(), i))\n",
    "# sort the stds\n",
    "train_patients = [x for x in combs[min(stds)[1]]]\n",
    "print(train_patients)\n",
    "print(images_df[images_df[\"patient_id\"].isin(train_patients)].groupby(\"score\").count()[\"patient_id\"])\n",
    "test_patients = [x for x in patients_ids if x not in train_patients]\n",
    "print(test_patients)\n",
    "print(images_df[images_df[\"patient_id\"].isin(test_patients)].groupby(\"score\").count()[\"patient_id\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of images for each score for patient 1045\n",
      "0    6198\n",
      "1      29\n",
      "2     370\n",
      "3     402\n",
      "Name: score, dtype: int64\n",
      "Number of images for each score for patient 1066\n",
      "0     210\n",
      "1     911\n",
      "2    1707\n",
      "3     677\n",
      "Name: score, dtype: int64\n",
      "Number of images for each score for patient 1068\n",
      "0     217\n",
      "1     781\n",
      "2    3458\n",
      "3    2552\n",
      "Name: score, dtype: int64\n",
      "348\n",
      "Number of images for each score for the selected images\n",
      "0    87\n",
      "1    87\n",
      "2    87\n",
      "3    87\n",
      "Name: score, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "for patient in test_patients:\n",
    "    # print the number of images for each score based on image_df\n",
    "    print(\"Number of images for each score for patient\", patient)\n",
    "    result = images_df[images_df[\"patient_id\"] == patient][\"score\"].value_counts()\n",
    "    # sort the result by score\n",
    "    result = result.sort_index()\n",
    "    print(result)\n",
    "\n",
    "max_number = 29\n",
    "selected_df_test = pd.DataFrame()\n",
    "for patient in test_patients:\n",
    "    for score in range(0,4):\n",
    "        # get the images for the patient and score\n",
    "        patient_score_df = images_df[(images_df[\"patient_id\"] == patient) & (images_df[\"score\"] == f\"{score}\")]\n",
    "        # if the number of images is less than 40, select all of them\n",
    "        if len(patient_score_df) < max_number:\n",
    "            selected_df_test = pd.concat([selected_df_test, patient_score_df])\n",
    "        else:\n",
    "            # select 29 images randomly\n",
    "            selected_df_test = pd.concat([selected_df_test, patient_score_df.sample(n=max_number, random_state=42)])\n",
    "print(len(selected_df_test))\n",
    "# print the number of images for each score based on selected_df\n",
    "print(\"Number of images for each score for the selected images\")\n",
    "result = selected_df_test[\"score\"].value_counts()\n",
    "# sort the result by score\n",
    "result = result.sort_index()\n",
    "print(result)\n",
    "\n",
    "# create a new dataframe using each entry of the selected_df as path\n",
    "paths_test_df = pd.DataFrame()\n",
    "for index, row in selected_df_test.iterrows():\n",
    "    # get the path\n",
    "    path = os.path.join(\"images/\", row[\"patient_id\"] + \"_\" + row[\"exam_id\"] + \"_\" + row[\"spot\"] + \"_\" + row[\"frame_number\"] + \"_\" + row[\"score\"] + \".png\" )\n",
    "    # create a new dataframe with the path\n",
    "    tmp_df = pd.DataFrame({\"path\": [path]})\n",
    "    # add the new dataframe to the values dataframe\n",
    "    paths_test_df = pd.concat([paths_test_df, tmp_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getPath(row):\n",
    "    return os.path.join(\"images/\", row[\"patient_id\"] + \"_\" + row[\"exam_id\"] + \"_\" + row[\"spot\"] + \"_\" + row[\"frame_number\"] + \"_\" + row[\"score\"] + \".png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_images_tmp = images_df[images_df[\"patient_id\"].isin(test_patients)]\n",
    "test_images_df = test_images_tmp.reset_index(drop=True)\n",
    "test_images_df[\"path\"] = test_images_df.apply(getPath, axis=1)\n",
    "test_images_df = test_images_df[test_images_df[\"path\"].isin(paths_test_df[\"path\"])]\n",
    "test_dataset = ImageDataset(data_dir, test_patients, test_images_df)\n",
    "\n",
    "values_train = pd.read_csv(\"data/values_train.csv\")\n",
    "values_test = pd.read_csv(\"data/values_test.csv\")\n",
    "values_test = values_test[values_test[\"path\"].isin(paths_test_df[\"path\"])]\n",
    "values_test = values_test.drop(values_test[values_test[\"correct\"] == 1].sample(n=48, random_state=42).index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 348/348 [00:12<00:00, 26.80it/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf8AAAGdCAYAAAAczXrvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAzC0lEQVR4nO3de1xUdf7H8fegMCAKiCiIiuKmiXdDQ9LUFDMz09XKWtus7GZoKpVG91oLtVrNRC0zzV9rlrVatquuS+VlEy+Y5SXJO6aCd1CUkYDfH7lsc7R0auSM57yePc7j4XzPmXM+86CZz3w+53vOOMrKysoEAABsw8/sAAAAQMUi+QMAYDMkfwAAbIbkDwCAzZD8AQCwGZI/AAA2Q/IHAMBmSP4AANgMyR8AAJupbHYA/9V10iqzQ8BZY29qanYIOOuKyKpmh4CzsnKOmR0CfqZ7XMQl3X9Qm6Fe29fpryd7bV/e4jPJHwAAn+GwdmPc2q8OAACcg8ofAAAjh8PsCC4pkj8AAEYWb/uT/AEAMLJ45W/trzYAAOAcVP4AABjR9gcAwGZo+wMAACuh8gcAwIi2PwAANkPbHwAAWAmVPwAARhZv+1v71QEA8Fs4HN5bPLRv3z7deeedqlGjhoKCgtSiRQutW7eufH1ZWZmeffZZ1a5dW0FBQUpKStK2bds8OgbJHwAAH3Hs2DF16NBB/v7+WrRokbZs2aLXXntN1atXL99m/PjxmjRpkqZNm6bVq1crODhYPXr0UFFR0UUfh7Y/AABGJrX9x40bp3r16mnmzJnlY7GxseX/Lisr08SJE/X000+rT58+kqTZs2crMjJSCxYs0O23335Rx6HyBwDAyIttf5fLpYKCArfF5XKd97Cffvqp2rZtq1tvvVW1atVSmzZtNH369PL1u3btUm5urpKSksrHQkNDlZCQoFWrVl30yyP5AwBg5PDz2pKWlqbQ0FC3JS0t7byH3blzp6ZOnapGjRppyZIlGjJkiB555BG9++67kqTc3FxJUmRkpNvzIiMjy9ddDNr+AABcQqmpqUpJSXEbczqd5922tLRUbdu21csvvyxJatOmjTZt2qRp06Zp0KBBXouJyh8AACMvVv5Op1MhISFuyy8l/9q1a6tp06ZuY3FxccrJyZEkRUVFSZLy8vLctsnLyytfdzFI/gAAGPk5vLd4oEOHDsrOznYb+/7771W/fn1JP03+i4qKUkZGRvn6goICrV69WomJiRd9HNr+AAD4iJEjR+qaa67Ryy+/rNtuu01r1qzRW2+9pbfeekuS5HA4NGLECI0ZM0aNGjVSbGysnnnmGUVHR6tv374XfRySPwAARiZd6teuXTvNnz9fqampevHFFxUbG6uJEydq4MCB5duMGjVKhYWFeuCBB3T8+HF17NhRixcvVmBg4EUfx1FWVlZ2KV6Ap7pOuvhLFHBpjb2p6YU3QoW4IrKq2SHgrKycY2aHgJ/pHhdxSfcf1O1lr+3rdMaTXtuXt3DOHwAAm6HtDwCAkcV/2IfkDwCA0W/4QZ7LibW/2gAAgHNQ+QMAYETbHwAAm7F425/kDwCAkcUrf2u/OgAAcA4qfwAAjGj7AwBgM7T9AQCAlVD5AwBgRNsfAACboe0PAACshMofAAAji1f+JP/f4Y74aN3fob4+/vqA0lfsliT1alZL3a6MUKNawQoOqKze09ao8EyJuYFa1NaN6/WPj97T7u1bdfzoYQ1/ZrzaXtOlfH3R6VP6YGa6sr5appMn8lUzMlrX97lN3Xr1Ny9om/hjryTlHth/zni/W+/Q46nPmBCRfSz5aLa+yVymvB/2yN/pVMMrW6jPoCGKrFO/fJuVSz7RuuVL9cPObBWdPqXx7y1WlarVTIzaB3HOH+dzZa1g3dQ8UjsOFbqNB/r7ae2e41q757ju71D/F54Nb3AVFSmmYSN1vr63Xh8z+pz1f3trorZ8s05DRr2giMja2pi1Wu+mj1f1GjV1VftOJkRsH++896FKS/73pXfHjm0aPuQ+devew8So7GH75g3q1LOf6jeKU0lJiRa+96YmPz9ST7/xNzkDgyRJxa4iNb0qQU2vStCn/zfN5IhhBpL/bxDo76cnezTSa5/v1J3t6rit+3hDriSpVZ0QM0KzlVbtrlGrdtf84vpt332ra5N6Ka5lvCSp641/1BeL5mtH9maS/yVWvXq42+PZM99Wnbr11Ca+nUkR2Ufyc391e3znI08pddBN2rsjW1c0ay1Juu7mAZKk7zeur+jwLh8Wb/tb+9VdIsO7xGr17mNavzff7FDwKxrFtdT6zOU6evigysrKtOWbdcrdl6MWVyWYHZqtFBef0ZJFC3VTn35yWLyV6ouKTv3UnaxSlYLEIw6H9xYf5HHlf/jwYb3zzjtatWqVcnN/qnKjoqJ0zTXX6O6771bNmjW9HqQvua5RDTWqWVVDPvjW7FBwAXcNeUzvTHpZw/98kypVqiSHw0+Dhz+pJi2uMjs0W1n2RYZOnjihXjf/0exQbKe0tFQfzXhdDeNaKrp+Q7PDubxYvPL3KPmvXbtWPXr0UJUqVZSUlKTGjRtLkvLy8jRp0iSNHTtWS5YsUdu2bX91Py6XSy6Xy22s9Mcz8qsc4GH4Fatm1QAld26gUfO/U3FJmdnh4AL+9emH2r51k0Y+95oiIqOUvfFrvTvlFYXVqKnmba42Ozzb+GzB39X+mmtVs2Yts0OxnQ/fek0H9uzUyLSpZocCH+NR8h82bJhuvfVWTZs27Zz2XVlZmR566CENGzZMq1at+tX9pKWl6YUXXnAba3DDvYrteZ8n4VS4xrWCFV4lQG/e0bJ8rJKfQy3rhKhvqyj1SM9UKd8JfMIZV5HmvTtFI54Zr9ZXd5QkxcQ20p6d3+ufH79H8q8gB/bv09o1q5T26utmh2I7H771mjat/UojXk5X9Qi+eHnMR9v13uJR8v/mm280a9as8563czgcGjlypNq0aXPB/aSmpiolJcVt7Oa3v/YkFFOs35uve9/b4DY2qvsV2nvstN5ft4/E70NKfvxRJT/+KIehdefnV0ll/KEqzD8+na/q4eG6pmNns0OxjbKyMs2b/ld9k7lcw8dMVkRktNkhXZasPj/Fo+QfFRWlNWvWqEmTJuddv2bNGkVGRl5wP06nU06n023M11v+knS6uFS7j552GysqLlHB6R/Lx6tX8Vd4FX/VCQuUJDWMqKJTZ0p08MQZnXD9WOExW1nR6VPK2/9D+eNDefu1Z8f3Cq4WoohaUWrS4iq9P2OSApxO1agVpa0bv9bKjH/qT/cPNzFq+ygtLdU/Pp2vG2/qq8qVubCoonz45mtat3ypHnhyrAKDqqjg2BFJUmCVqgo4+7lbcOyICo4d0eHcn94/+/fsUGBQFVWvGaXgakwMtAOP3pGPPfaYHnjgAWVlZalbt27liT4vL08ZGRmaPn26Xn311UsS6OXi5haRGpRQr/zx67c0lySNW7pdS747ZFZYlrRr23d6efSQ8sdz3pooSeqY1EsPPvqckp8Yow9nTdHU8c/q5IkCRdSK0q2DHuImPxVk7epVys09oJv69DM7FFtZsXi+JOn1p4e6jd857Em179br7DYLtOiDd8rXTXwq+Zxt7M7qlb+jrKzMox7oBx98oAkTJigrK0slZ2/iUalSJcXHxyslJUW33Xbbbwqk66RfnyeAijP2pqZmh4CzroisanYIOCsr55jZIeBnusdFXNL9B98602v7Kpx3j9f25S0e9+IGDBigAQMGqLi4WIcPH5YkRUREyN/f3+vBAQAA7/vNJ+L8/f1Vu3Ztb8YCAIBPsHrbn1k4AAAYWD35W/sWRgAA4BxU/gAAGFi98if5AwBgQPIHAMBurJ37OecPAIDdUPkDAGBA2x8AAJuxevKn7Q8AgM1Q+QMAYGD1yp/kDwCAgdWTP21/AABshsofAAAjaxf+JH8AAIxo+wMAAEuh8gcAwMDqlT/JHwAAA5I/AAB2Y+3czzl/AADshsofAAAD2v4AANiM1ZM/bX8AAGyGyh8AAAOrV/4kfwAADKye/Gn7AwDgI55//nk5HA63pUmTJuXri4qKlJycrBo1aqhq1arq37+/8vLyPD4OyR8AACOHFxcPNWvWTAcOHChfVq5cWb5u5MiRWrhwoebNm6dly5Zp//796tevn8fHoO0PAICBmW3/ypUrKyoq6pzx/Px8zZgxQ3PmzFHXrl0lSTNnzlRcXJwyMzPVvn37iz4GlT8AAJeQy+VSQUGB2+JyuX5x+23btik6OloNGzbUwIEDlZOTI0nKyspScXGxkpKSyrdt0qSJYmJitGrVKo9iIvkDAGBgPO/+e5a0tDSFhoa6LWlpaec9bkJCgmbNmqXFixdr6tSp2rVrl6699lqdOHFCubm5CggIUFhYmNtzIiMjlZub69Hro+0PAICBN9v+qampSklJcRtzOp3n3bZnz57l/27ZsqUSEhJUv359ffjhhwoKCvJaTCR/AACMvHjK3+l0/mKyv5CwsDA1btxY27dvV/fu3XXmzBkdP37crfrPy8s77xyBX0PbHwAAH3Xy5Ent2LFDtWvXVnx8vPz9/ZWRkVG+Pjs7Wzk5OUpMTPRov1T+AAAYmDXb/7HHHlPv3r1Vv3597d+/X88995wqVaqkO+64Q6GhoRo8eLBSUlIUHh6ukJAQDRs2TImJiR7N9JdI/gAAnMOs5P/DDz/ojjvu0JEjR1SzZk117NhRmZmZqlmzpiRpwoQJ8vPzU//+/eVyudSjRw9NmTLF4+OQ/AEA8BFz58791fWBgYFKT09Xenr67zoOyR8AAAOr39uf5A8AgIHVkz+z/QEAsBkqfwAAjKxd+PtO8r8rsY7ZIeCsO9K/MjsEnPWfZ5MuvBEqxLWNIswOARWItj8AALAUn6n8AQDwFVav/En+AAAYWDz3k/wBADCyeuXPOX8AAGyGyh8AAAOLF/4kfwAAjGj7AwAAS6HyBwDAwOKFP8kfAAAjPz9rZ3/a/gAA2AyVPwAABrT9AQCwGWb7AwAAS6HyBwDAwOKFP8kfAAAjq7f9Sf4AABhYPflzzh8AAJuh8gcAwMDihT/JHwAAI9r+AADAUqj8AQAwsHjhT/IHAMCItj8AALAUKn8AAAwsXviT/AEAMKLtDwAALIXKHwAAA4sX/iR/AACMrN72J/kDAGBg8dzPOX8AAOyGyh8AAAPa/gAA2IzFcz9tfwAA7IbKHwAAA9r+AADYjMVzP21/AADshsofAAAD2v4AANiM1ZM/bX8AAGyGyt9DX336vrLXrtTRA3tVOcCpOo2a6roB96lGdD237X7YtkXL583U/h1b5XD4KbL+HzRgdJr8A5wmRW5tD3ZtqFG9rtTM5bs15pPvJEkR1QL0xE1N1LFxhIKdlbTzUKGm/HuHlmzMMzlaezhVWKgZb76hFV9m6Nixo2rUuImGPfqE4pq2MDs0W5sx/S1NmviaBt55l0alPmV2OD7L4oU/yd9TOd99q/juN6t2wytVWlKiZR++o7njntD9495WQGCQpJ8S/4fjU5XY+w51vytZfn6VdDBnp+XbSGZpUS9Ud7Svp+/2F7iNv3pHK4UEVdYD72TpWOEZ3XxVtN64q436TvxKW/YV/MLe4C3jX3pWu3Zs11PPp6lGzVpaumihHk2+X+9+8Ilq1oo0Ozxb2rTxW300b64aN77S7FB8ntU/r2n7e+j20Wlq2amHatZtoMj6f9BNDz6ugiMHlbt7W/k2Ge9NVfz1f1TizberZt0GqhFdT3HtO6uyf4CJkVtTlYBKmjCwlZ6ct0n5p4rd1l3VIEyzV+7Rt3vztffoaaX/e4cKThered0Qk6K1D1dRkZZ/8W89NCxFra5qq7r1YnTPA8mqUy9Gn3z8gdnh2dKpwkKljn5cz70wRiGhoWaH4/McDu8tvojk/zsVnSqUJAUFV5MkFeYf0/4dWxUcGqbZLwzX6w/fqvfGpGhv9iYzw7SsF/o11RdbDuqrbUfOWbd+93H1al1boUH+cjikm1rXlrOyn1ZvP2pCpPZSUlKikpISBRhOczmdTm38Zr1JUdnby2NeVKdOndU+8RqzQ4EP8Hry37t3r+69995f3cblcqmgoMBtKT7j8nYol1xZaan+/d5U1W3cTDXrxUqSjh86IEla8ffZat2lpwaMSlNUg0Z6P22Ujub+YGa4lnNT69pqVjdUr/zz+/OuHzb7a1Wu5ND6MUn6blwPjbmlmYbM+lp7jpyq4Ejtp0pwsJq1aKXZ70zT4UMHVVJSon8tWqjNG7/RkcOHzQ7Pdhb98x/67rstemTko2aHctlwOBxeW3yR15P/0aNH9e677/7qNmlpaQoNDXVb/jFrirdDueSWvPuGDv+wW32S/zdppqy0TJLU5rpeatn5BkU1uEJJdw5ReO26+nbZErNCtZzaYYF6pm+cRv7tG535sfS826T0bKSQQH/9edoa9Z3wlWYs36037mqtxlFVKzhae3rqhTSVlUn9e3VV945X6eMP/qZu1/eUw883PwytKvfAAY0f+5LSxr0ip5MJxxfL6m1/jyf8ffrpp7+6fufOnRfcR2pqqlJSUtzGPrjMZmAvefcNbf96te58+jWF1KhZPl41LFySFFGnvtv2EdExyj9ysEJjtLLmdUMUUc2pT0f+r4VZuZKfrm4Yrj93iFH3cSt0V8cGumH8Cm3LOylJ2nrghNrFVtefO9TXMx9vNit026hTN0aT3pyl06dP6VRhoWpE1NTzTz6q6Dp1zQ7NVrZs2ayjR47o9lv7lY+VlJQoa91azX3/b1r79UZVqlTJxAhhBo+Tf9++feVwOFRWVvaL21yozeF0Os/5BuofcNzTUExRVlamf82erO/X/UcDn3pVYbVqu60PrRmlqtVr6MgB9xb/0dwf1LBlu4oM1dK+2nZEPV9Z4TY2bkAL7ThYqLe+2KlA/5+aWqWG/09LyspE4VmxgoKqKCioik4U5Gtt5ld6cFjKhZ8Er0lo314fLVjoNvbcU6lq0LCh7hl8P4n/F/j5QMk+duxYpaamavjw4Zo4caIkqaioSI8++qjmzp0rl8ulHj16aMqUKYqM9OwKGo/b/rVr19bf//53lZaWnndZv97ak3mWzHpDm/+ToT4PpyogsIpOHj+qk8ePls9ZcDgcSuh1m7L+NV9b1yzX0dx9WjZvlo7s36tWXXqaHL11FLpK9H3uSbfl1JkSHT9VrO9zT2rnwULtPlSoMbc0V8t6oYqpUUWDOzdQx0YRWrqJDkxFWLPqP1q9aqUO7PtBa1d/pRFD7lVMg1jd2Luv2aHZSnBwVTVq1NhtCapSRWGhYWrUqLHZ4fkss9v+a9eu1ZtvvqmWLVu6jY8cOVILFy7UvHnztGzZMu3fv1/9+vX7hb38Mo8r//j4eGVlZalPnz7nXX+hrsDl7uuMn75B/+2lx9zGez3wmFp26iFJuvqGfio5c0b/fm+aigpPqFZMQ93+xDhVj4yu8Hjt6sfSMg1+e50e73Wlpg+OV5WAStpz5JQen/utvtx6yOzwbOHkyROaPmWiDh3MU7WQUHXu2l33DXlElSv7mx0a4NNOnjypgQMHavr06RozZkz5eH5+vmbMmKE5c+aoa9eukqSZM2cqLi5OmZmZat++/UUfw1HmYaZesWKFCgsLdcMNN5x3fWFhodatW6fOnTt7slvNWpvj0fa4dP4yl/PhvuI/zyaZHQLOCgvmS4svCbzEt6jrMWW11/b16eDWcrncr2g73+nv/xo0aJDCw8M1YcIEdenSRa1bt9bEiRP1+eefq1u3bjp27JjCwsLKt69fv75GjBihkSNHXnRMHrf9r7322l9M/JIUHBzsceIHAMCX+Dm8t5zvCre0tLTzHnfu3Llav379edfn5uYqICDALfFLUmRkpHJzcz16fdzeFwAAA29en3++K9zOV/Xv3btXw4cP19KlSxUYGOi1458PyR8AgEvo11r8P5eVlaWDBw/qqquuKh8rKSnR8uXLNXnyZC1ZskRnzpzR8ePH3ar/vLw8RUVFeRQTyR8AAAMzrvTr1q2bNm7c6DZ2zz33qEmTJho9erTq1asnf39/ZWRkqH///pKk7Oxs5eTkKDEx0aNjkfwBADBwqOKzf7Vq1dS8eXO3seDgYNWoUaN8fPDgwUpJSVF4eLhCQkI0bNgwJSYmejTTXyL5AwBw2ZgwYYL8/PzUv39/t5v8eIrkDwCAga/cCfTLL790exwYGKj09HSlp6f/rv2S/AEAMPDVX+PzFq//qh8AAPBtVP4AABhYvPAn+QMAYOQLv+p3KdH2BwDAZqj8AQAwsHjhT/IHAMDI6rP9Sf4AABhYPPdzzh8AALuh8gcAwMDqs/1J/gAAGFg79dP2BwDAdqj8AQAwYLY/AAA24yu/6nep0PYHAMBmqPwBADCg7Q8AgM1YPPfT9gcAwG6o/AEAMKDtDwCAzVh9tj/JHwAAA6tX/pzzBwDAZqj8AQAwsHbdT/IHAOAcVv9VP9r+AADYDJU/AAAGFi/8Sf4AABgx2x8AAFgKlT8AAAYWL/xJ/gAAGDHbHwAAWAqVPwAABhYv/En+AAAYWX22v88k/4Q6NcwOAWcteryz2SHgrKRXvjQ7BJw17+FrzA4BPxMXHXxJ92/1c+JWf30AAMDAZyp/AAB8BW1/AABsxs/auZ+2PwAAdkPlDwCAgdUrf5I/AAAGVj/nT9sfAACbofIHAMCAtj8AADZj8a4/bX8AAOyGyh8AAAOr/6QvyR8AAAOrt8VJ/gAAGFi88Lf8lxsAAGBA5Q8AgAHn/AEAsBmL537a/gAA2A3JHwAAAz+H9xZPTJ06VS1btlRISIhCQkKUmJioRYsWla8vKipScnKyatSooapVq6p///7Ky8vz/PV5/AwAACzOz+Hw2uKJunXrauzYscrKytK6devUtWtX9enTR5s3b5YkjRw5UgsXLtS8efO0bNky7d+/X/369fP49XHOHwAAH9G7d2+3xy+99JKmTp2qzMxM1a1bVzNmzNCcOXPUtWtXSdLMmTMVFxenzMxMtW/f/qKPQ/IHAMDAmxP+XC6XXC6X25jT6ZTT6fzV55WUlGjevHkqLCxUYmKisrKyVFxcrKSkpPJtmjRpopiYGK1atcqj5E/bHwAAA2+e809LS1NoaKjbkpaW9ovH3rhxo6pWrSqn06mHHnpI8+fPV9OmTZWbm6uAgACFhYW5bR8ZGanc3FyPXh+VPwAAl1BqaqpSUlLcxn6t6r/yyiu1YcMG5efn66OPPtKgQYO0bNkyr8ZE8gcAwMAh7/X9L6bF/3MBAQG64oorJEnx8fFau3atXn/9dQ0YMEBnzpzR8ePH3ar/vLw8RUVFeRQTbX8AAAzMutTvfEpLS+VyuRQfHy9/f39lZGSUr8vOzlZOTo4SExM92ieVPwAABt5I2r9FamqqevbsqZiYGJ04cUJz5szRl19+qSVLlig0NFSDBw9WSkqKwsPDFRISomHDhikxMdGjyX4SyR8AAJ9x8OBB3XXXXTpw4IBCQ0PVsmVLLVmyRN27d5ckTZgwQX5+furfv79cLpd69OihKVOmeHwckj8AAAYOk27uP2PGjF9dHxgYqPT0dKWnp/+u45D8AQAwMKvtX1GY8AcAgM1Q+QMAYGD1n/Ql+QMAYODpD/Jcbmj7AwBgM1T+AAAYWH3CH8kfAAADi3f9afsDAGA3VP4AABj4efGHfXwRyR8AAAOrt/1J/gAAGFh9wh/n/AEAsBmSv4c2f5OlMU8O1z23XK++112lzJVf/OK2U//6kvped5U+/ehvFRihfWz6JksvPjFcg/p1V+/ObbRqhfvfYs7MaXroz3/ULT0SdXuvTno65UFlb9loUrT2MrhTA20a012jb2zsNt6qXqhm3BuvNc92VebT12nWfW3lrMzHkLfxOfX7+TkcXlt8Ee86DxUVFSn2D4314PAnfnW7zBWfK3vLRoVH1KygyOyn6PRpxV7RWA+NSD3v+ui69fXQ8NGaPHOexk2eqVpR0Xr2sYeVf/xoBUdqL83rhOjWdnWVfeCE23ireqGaNqiNvtp+RHdMW63bp63W+5l7VVpWZlKk1sXn1O/ncHhv8UWc8/dQfEIHxSd0+NVtjhw6qOmTxuu58en6S+ojFRSZ/bRt31Ft23f8xfVduvd0e3xf8qNa+o8F2r1jm1rFJ1zq8GwpKKCSxt7aXM8v2KIHu8S6rRt1Y2P9bdVezVi+u3xs9+FTFRyhPfA5hQuh8vey0tJSTUx7Wn0H3KWY2D+YHQ7OKi4u1uKFf1dw1apq8IfGF34CfpOnezfR8uzDytzh3l0JD/ZXq3phOlp4Ru890E7LnuikmYPbqk39MHMCtTk+py6Mtr/B6dOntXLlSm3ZsuWcdUVFRZo9e7ZXArtc/f39WfKrVFk39b/D7FAgac1Xy3XrDdeof/cEfTLvPb346jSFhlU3OyxL6tkiUnG1q2ni0u3nrKtbvYok6eGuDfXRun168N2v9d3+As24J14xNapUdKi2x+fUhVm97e9R8v/+++8VFxenTp06qUWLFurcubMOHDhQvj4/P1/33HPPBffjcrlUUFDgtpxxuTyP3sdsz96izz5+X8NHvyCHr/7FbaZlm3Z6/e25Gp8+S/FXX6Nxz4/S8WOc8/e2qFCnnuh1pZ6Yt0lnfiw9Z/1/L5uat3afFqzfr60HTmj8ou+1+3Ch+l0VXcHR2hufU5A8TP6jR49W8+bNdfDgQWVnZ6tatWrq0KGDcnJyPDpoWlqaQkND3Za3Jr/q0T580ZaNXyv/+FHdN+BG9evWTv26tdOhvAOaNXWC7r+9l9nh2VJgUJCi68aoSbOWemT086pUqZKW/mO+2WFZTtPoENWo6tSHDydowwvdtOGFbmoXG66B7WO04YVuOlJ4RpK04+BJt+ftPFSoqLBAM0K2LT6nLo6fFxdf5NGEv6+++kr//ve/FRERoYiICC1cuFAPP/ywrr32Wn3xxRcKDg6+qP2kpqYqJSXFbWzXkR89CcUndene65yJZC+MSlaX7r3U7YabTYoKP1dWVqbi4mKzw7CczB1H1XfSV25jY/o1067DhZqxfLf2Hj2tvIIiNYhw/4yoX6OKVm47UpGh2h6fUxfH6l0Rj5L/6dOnVbny/57icDg0depUDR06VJ07d9acOXMuaj9Op1NOp9NtLOBkoSehmOb06VM6sG9v+eODB/Zp5/ZsVasWopqRtRUSGua2faVKlRUWXkN1YhpUbKA2cPqU+98i78A+7dyWraohIQoJCdOH//e2ru7QWeE1IlSQf1z/mP+hjhw+qA5dupsYtTWdOlOi7Qfd38Oni0t0/FRx+fjMFXuU3K2hsnNPaOuBE+rTJlqxNYOVMvdbM0K2ND6ncCEeJf8mTZpo3bp1iouLcxufPHmyJOnmm63/rXF79hY9M/KB8sfvTPmrJOm6Hr01/IkXzArLlrZnb9GTI+4vfzwj/TVJUtcbeis55Sn9kLNbGUsWqiD/uEJCQtWoSTONnfSO6jO72RTvrcqR099Po2+8UiFB/vo+94Tun7Vee4+eNjs0y+Fz6vezdt0vOcrKLv4OG2lpaVqxYoX++c9/nnf9ww8/rGnTpqm09NwJPxfy3f7Lo/K3g0pWv6n1ZaTf5P+YHQLOmvfwNWaHgJ+Ji76408y/1XtZP3htX3fG1/XavrzFo7kIqampv5j4JWnKlCm/KfEDAOBLHF5cfJGvTkQEAACXCLf3BQDAwOKT/Un+AAAYWf1SP9r+AADYDJU/AAAGVq+MSf4AABjQ9gcAAJZC5Q8AgIG1636SPwAA56DtDwAALIXKHwAAA6tXxiR/AAAMrN72J/kDAGBg7dRv/c4GAAAwoPIHAMDA4l1/kj8AAEZ+Fm/80/YHAMBmqPwBADCg7Q8AgM04aPsDAAArofIHAMCAtj8AADbDbH8AAGApVP4AABjQ9gcAwGZI/gAA2AyX+gEAAEuh8gcAwMDP2oU/lT8AAEYOL/7nibS0NLVr107VqlVTrVq11LdvX2VnZ7ttU1RUpOTkZNWoUUNVq1ZV//79lZeX59FxSP4AAPiIZcuWKTk5WZmZmVq6dKmKi4t1/fXXq7CwsHybkSNHauHChZo3b56WLVum/fv3q1+/fh4dh7Y/AAAGZs32X7x4sdvjWbNmqVatWsrKylKnTp2Un5+vGTNmaM6cOerataskaebMmYqLi1NmZqbat29/Uceh8gcAwMCbbX+Xy6WCggK3xeVyXVQc+fn5kqTw8HBJUlZWloqLi5WUlFS+TZMmTRQTE6NVq1Zd9Osj+QMAcAmlpaUpNDTUbUlLS7vg80pLSzVixAh16NBBzZs3lyTl5uYqICBAYWFhbttGRkYqNzf3omOi7Q8AgIE3Z/unpqYqJSXFbczpdF7wecnJydq0aZNWrlzpvWDOIvkDAGDgzZv8OJ3Oi0r2Pzd06FB99tlnWr58uerWrVs+HhUVpTNnzuj48eNu1X9eXp6ioqIuev+0/QEA8BFlZWUaOnSo5s+fr88//1yxsbFu6+Pj4+Xv76+MjIzysezsbOXk5CgxMfGij0PlDwCAgVmz/ZOTkzVnzhx98sknqlatWvl5/NDQUAUFBSk0NFSDBw9WSkqKwsPDFRISomHDhikxMfGiZ/pLJH8AAM5h1g3+pk6dKknq0qWL2/jMmTN19913S5ImTJggPz8/9e/fXy6XSz169NCUKVM8Og7JHwAAAz+TSv+ysrILbhMYGKj09HSlp6f/5uNwzh8AAJtxlF3M14wKkJtfbHYIOKuKs5LZIeCsU64Ss0PAWbFdRpodAn7m9NeTL+n+M7cf99q+2l8R5rV9eQttfwAAjPhVPwAAYCVU/gAAGHjzJj++iOQPAICBWdf5VxTa/gAA2AyVPwAABhYv/En+AACcw+LZn7Y/AAA2Q+UPAIABs/0BALAZq8/2J/kDAGBg8dzPOX8AAOyGyh8AACOLl/4kfwAADKw+4Y+2PwAANkPlDwCAAbP9AQCwGYvnftr+AADYDZU/AABGFi/9Sf4AABgw2x8AAFgKlT8AAAbM9gcAwGYsnvtJ/gAAnMPi2Z9z/gAA2AyVPwAABlaf7U/yBwDAwOoT/mj7AwBgM1T+AAAYWLzwJ/kDAHAOi2d/2v4AANgMlT8AAAbM9gcAwGaY7Q8AACyFyh8AAAOLF/4kfwAAzmHx7E/yBwDAwOoT/jjnDwCAzVD5AwBgYPXZ/iR/LzhVWKgZb76hFV9m6Nixo2rUuImGPfqE4pq2MDs02zmYl6c3Jr6mVf9ZrqKiItWtF6NnX3xZTZs1Nzs02+F9YZ7omqEaM7yPru/QTFUC/bVj72E9+Px7Wr8lR5L01IM36tYeV6luVHWdKS7R19/l6PnJC7V20x6TI/cdFs/9JH9vGP/Ss9q1Y7ueej5NNWrW0tJFC/Vo8v1694NPVLNWpNnh2UZBQb7uu/tPim+boNfT31JY9XDtzdmjkJAQs0OzJd4X5girFqTPZ6Vo2dpt6jt0ig4dO6krYmrqWMGp8m227zmokePmadcPhxXk9NewO7tq4ZShat7nBR0+dtLE6FFRHGVlZWVmByFJufnFZofwm7iKitTzugS99MokJXbsXD5+/123KSGxo+4b8oiJ0f02VZyVzA7hN3lj4mv6dsPXmj7rPbND8ZpTrhKzQ/hNrPi+iO0y0uwQLspfHrlZia0aKmnwxIt+TrXgQB1c+ap6PjhJX675/tIF50Wnv558Sfe/49Bpr+3rDzWDvLYvb2HC3+9UUlKikpISBQQ43cadTqc2frPepKjsacWyLxTXrJmeeGyEru/SQQNv66f5H39odli2xPvCPL06t9D6LTn62/h7tScjTaveH617/njNL27vX7mSBvfroOMnTmnj9/sqMFLf5vDif76I5P87VQkOVrMWrTT7nWk6fOigSkpK9K9FC7V54zc6cviw2eHZyr4f9urjD+eqXkx9vTF1uvrfdrteG/eyPvt0gdmh2Q7vC/PE1onQ/bdeq+05h3Tzw+maPm+lXht1iwb2TnDbrue1zXXoP6/p+OoJGnbndbrpock6crzQpKhR0Txu+3/33XfKzMxUYmKimjRpoq1bt+r111+Xy+XSnXfeqa5du15wHy6XSy6Xy23sWJGfnE7nLzzDt+37IUfj/vKsvvl6nSpVqqRGV8apXkx9ZW/dov/7cKHZ4Xnscm37J8a3VFyzZnpn9vvlY6+OfUlbNm/UO/8318TIfrvLte0vWe99cbm0/fPXTNT6LTm67u6/lo+9NuoWxTerry6DXisfqxIYoKiaIYoIq6p7+l2jLu0aq9OfX9Why+Sc/6Vu++86XOS1fcVGBHptX97iUeW/ePFitW7dWo899pjatGmjxYsXq1OnTtq+fbv27Nmj66+/Xp9//vkF95OWlqbQ0FC35Y2/jvvNL8JsderGaNKbs7R42RrNW/hvvTlrrn788UdF16lrdmi2ElEzQg0b/sFtrEHDhso9cMCkiOyN94U5cg8X6LuduW5jW3flql5UdbexU0VntHPvYa3ZuFtDXpijH0tKNehXTg/YjcOLiy/yKPm/+OKLevzxx3XkyBHNnDlTf/rTn3T//fdr6dKlysjI0OOPP66xY8decD+pqanKz893W4aljP7NL8JXBAVVUY2ImjpRkK+1mV+pQ6cLd0HgPa1aX6U9u3e7jeXs2a2o6GhzAoIk3hcVbdWGnWpcv5bbWKOYWso5cPRXn+fncMjpzwVgduFR8t+8ebPuvvtuSdJtt92mEydO6JZbbilfP3DgQH377bcX3I/T6VRISIjbcrm2/CVpzar/aPWqlTqw7wetXf2VRgy5VzENYnVj775mh2Yrd9w5SBs3fqOZb7+pvTl7tPifn2n+R/N064A/mR2aLfG+MMcb732uq1vE6vF7r1fDehEacENb3du/g978YLmkn9r9LwztratbNFBM7epqE1dP054bqOhaYfr7UiZjlrN46e/x1zzH2dse+fn5KTAwUKGhoeXrqlWrpvz8fO9Fd5k4efKEpk+ZqEMH81QtJFSdu3bXfUMeUeXK/maHZivNmrfQK3+dpPRJE/T2m1MUXaeuUkY9oZ69epsdmi3xvjBH1pYcDXh0ul4cdrOefKCndu87osdf+VhzF62TJJWUlurKBpG6s3eCaoQF62j+Ka3bvEdJ904453SBnfnqLH1v8WjCX6tWrTRu3DjdcMMNkqRNmzapSZMmqlz5p+8QK1as0KBBg7Rz506PA7lcr/O3ost1wp8VXc4T/qzmcpnwZxeXesJfzlHXhTe6SDHhvtfZ9qjtP2TIEJWU/O/DqHnz5uWJX5IWLVp0UbP9AQDAuZYvX67evXsrOjpaDodDCxYscFtfVlamZ599VrVr11ZQUJCSkpK0bds2j4/jUfJ/6KGH1KtXr19c//LLL+vtt9/2OAgAAHyJWaf8CwsL1apVK6Wnp593/fjx4zVp0iRNmzZNq1evVnBwsHr06KGiIs8uTWRqJwAABmb9ql/Pnj3Vs2fP864rKyvTxIkT9fTTT6tPnz6SpNmzZysyMlILFizQ7bffftHH4Q5/AABcQi6XSwUFBW6L8UZ3F2PXrl3Kzc1VUlJS+VhoaKgSEhK0atUqj/ZF8gcA4Bzea/yf78Z2aWlpHkeUm/vT1RiRke6/ihkZGVm+7mLR9gcAwMCbbf/U1FSlpKS4jZl9bxuSPwAAl5DT6fRKso+KipIk5eXlqXbt2uXjeXl5at26tUf7ou0PAICBL97gLzY2VlFRUcrIyCgfKygo0OrVq5WYmOjRvqj8AQAwMGu2/8mTJ7V9+/byx7t27dKGDRsUHh6umJgYjRgxQmPGjFGjRo0UGxurZ555RtHR0erbt69HxyH5AwDgI9atW6frrruu/PF/5woMGjRIs2bN0qhRo1RYWKgHHnhAx48fV8eOHbV48WIFBnr2s8Ee3d73UuL2vr6D2/v6Dm7v6zu4va9vudS39/VmTooK9b3fs6DyBwDAyNq/60PyBwDAyOK5n9n+AADYDZU/AAAGZs32rygkfwAADBwWb/zT9gcAwGao/AEAMLJ24U/yBwDAyOK5n7Y/AAB2Q+UPAIABs/0BALAZZvsDAABLofIHAMDA6m1/Kn8AAGyGyh8AAAMqfwAAYClU/gAAGFh9tj/JHwAAA9r+AADAUqj8AQAwsHjhT/IHAOAcFs/+tP0BALAZKn8AAAyY7Q8AgM0w2x8AAFgKlT8AAAYWL/xJ/gAAnMPi2Z/kDwCAgdUn/HHOHwAAm6HyBwDAwOqz/R1lZWVlZgdhBS6XS2lpaUpNTZXT6TQ7HNvj7+E7+Fv4Dv4W+C+Sv5cUFBQoNDRU+fn5CgkJMTsc2+Pv4Tv4W/gO/hb4L875AwBgMyR/AABshuQPAIDNkPy9xOl06rnnnmMSjY/g7+E7+Fv4Dv4W+C8m/AEAYDNU/gAA2AzJHwAAmyH5AwBgMyR/AABshuTvJenp6WrQoIECAwOVkJCgNWvWmB2SLS1fvly9e/dWdHS0HA6HFixYYHZItpSWlqZ27dqpWrVqqlWrlvr27avs7Gyzw7KtqVOnqmXLlgoJCVFISIgSExO1aNEis8OCiUj+XvDBBx8oJSVFzz33nNavX69WrVqpR48eOnjwoNmh2U5hYaFatWql9PR0s0OxtWXLlik5OVmZmZlaunSpiouLdf3116uwsNDs0Gypbt26Gjt2rLKysrRu3Tp17dpVffr00ebNm80ODSbhUj8vSEhIULt27TR58mRJUmlpqerVq6dhw4bpiSeeMDk6+3I4HJo/f7769u1rdii2d+jQIdWqVUvLli1Tp06dzA4HksLDw/XKK69o8ODBZocCE1D5/05nzpxRVlaWkpKSysf8/PyUlJSkVatWmRgZ4Dvy8/Ml/ZRwYK6SkhLNnTtXhYWFSkxMNDscmKSy2QFc7g4fPqySkhJFRka6jUdGRmrr1q0mRQX4jtLSUo0YMUIdOnRQ8+bNzQ7HtjZu3KjExEQVFRWpatWqmj9/vpo2bWp2WDAJyR/AJZWcnKxNmzZp5cqVZodia1deeaU2bNig/Px8ffTRRxo0aJCWLVvGFwCbIvn/ThEREapUqZLy8vLcxvPy8hQVFWVSVIBvGDp0qD777DMtX75cdevWNTscWwsICNAVV1whSYqPj9fatWv1+uuv68033zQ5MpiBc/6/U0BAgOLj45WRkVE+VlpaqoyMDM6nwbbKyso0dOhQzZ8/X59//rliY2PNDgkGpaWlcrlcZocBk1D5e0FKSooGDRqktm3b6uqrr9bEiRNVWFioe+65x+zQbOfkyZPavn17+eNdu3Zpw4YNCg8PV0xMjImR2UtycrLmzJmjTz75RNWqVVNubq4kKTQ0VEFBQSZHZz+pqanq2bOnYmJidOLECc2ZM0dffvmllixZYnZoMAmX+nnJ5MmT9corryg3N1etW7fWpEmTlJCQYHZYtvPll1/quuuuO2d80KBBmjVrVsUHZFMOh+O84zNnztTdd99dscFAgwcPVkZGhg4cOKDQ0FC1bNlSo0ePVvfu3c0ODSYh+QMAYDOc8wcAwGZI/gAA2AzJHwAAmyH5AwBgMyR/AABshuQPAIDNkPwBALAZkj8AADZD8gcAwGZI/gAA2AzJHwAAmyH5AwBgM/8PxwWcOH+znMQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.5689655172413793\n",
      "precision: 0.571448955168922\n"
     ]
    }
   ],
   "source": [
    "# confusion matrix of the models/MIDmodel2.pt\n",
    "model = MIDmodel2()\n",
    "model.load_state_dict(torch.load(\"models/MIDmodel2.pt\"))\n",
    "model.eval()\n",
    "model.to(device)\n",
    "test_dataset = ImageDataset(data_dir, test_patients, test_images_df)\n",
    "y_true = []\n",
    "y_pred = []\n",
    "for i in tqdm(range(len(test_dataset))):\n",
    "    image, label, path = test_dataset[i]\n",
    "    image = image.unsqueeze(0)\n",
    "    image = image.to(device)\n",
    "    output = model(image)\n",
    "    _, predicted = torch.max(output.data, 1)\n",
    "    y_true.append(label)\n",
    "    y_pred.append(predicted.cpu().detach().numpy())\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "plt.show()\n",
    "\n",
    "print(\"accuracy:\", accuracy_score(y_true, y_pred))\n",
    "print(\"precision:\", precision_score(y_true, y_pred, average=\"macro\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.5433333333333333\n",
      "Precision: 0.5437710437710438\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf8AAAGdCAYAAAAczXrvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgcklEQVR4nO3de3hU5bn38d8khgmIRCAkkyBICCIYUCIn8VAPRENkI2wRpdX9BuPlARE2RFHTV0BEDFFfZSsWq9umoEDFVqm2VWuioFROohDPnIkKCedEAgyQzPuHe0+dJwgZmWRNn/X9eK0/smbNWvdwNf3lvtczM55AIBAQAABwjRinCwAAAE2L8AcAwGUIfwAAXIbwBwDAZQh/AABchvAHAMBlCH8AAFyG8AcAwGUIfwAAXOYUpwv4X80z73K6BCDq7F01y+kSgKgU38jpFclMOvhJ9P0eR034AwAQNTx2D8btfnUAAKAeOn8AAEwej9MVNCrCHwAAk+Vjf8IfAACT5Z2/3X/aAACAeuj8AQAwMfYHAMBlGPsDAACb0PkDAGBi7A8AgMsw9gcAADah8wcAwMTYHwAAl2HsDwAAbELnDwCAibE/AAAuY/nYn/AHAMBkeedv96sDAAD10PkDAGCyvPMn/AEAMMXYfc/f7j9tAABAPXT+AACYGPsDAOAylr/Vz+4/bQAAQD10/gAAmBj7AwDgMoz9AQCATej8AQAwMfYHAMBlLB/7E/4AAJgs7/ztfnUAAKAeOn8AAEyM/QEAcBnG/gAAwCZ0/gAAmBj7AwDgMoz9AQCATej8AQAwWd75E/4AAJgsv+dv9582AACgHjp/AABMjP0BAHAZy8f+hD8AACbLO3+7Xx0AAKiHzh8AABNjfwAA3MVjefgz9gcAIErU1tZq0qRJSktLU/PmzZWenq5p06YpEAgEjxk1apQ8Hk/INmjQoLCuQ+cPAIDBqc6/qKhIs2fP1pw5c5SRkaGPPvpIN998sxISEjRu3LjgcYMGDVJxcXHwZ6/XG9Z1CH8AAEwOTf0//PBDDR06VIMHD5YkderUSQsWLNDKlStDjvN6vfL5fD/7Ooz9AQBoRH6/X9XV1SGb3+8/5rEXXnihSktLtW7dOknS2rVrtXTpUuXk5IQct3jxYiUlJenss8/W6NGjtXv37rBqIvwBADCY99RPZissLFRCQkLIVlhYeMzr3n///Ro5cqS6deumuLg4ZWZmavz48brxxhuDxwwaNEhz585VaWmpioqKtGTJEuXk5Ki2trbBr4+xPwAAhkje8y8oKFB+fn7Ivp+6R79w4ULNmzdP8+fPV0ZGhtasWaPx48crNTVVubm5kqSRI0cGj+/Zs6fOPfdcpaena/HixRo4cGCDaiL8AQBoRF6vt8EL8iZOnBjs/qUfwn3r1q0qLCwMhr+pc+fOSkxM1IYNGwh/AAB+LqdW+x84cEAxMaF35GNjY1VXV/eTz/n222+1e/dupaSkNPg6hD8AAAanwn/IkCGaPn26OnbsqIyMDH3yySd64oknlJeXJ0nav3+/pk6dquHDh8vn82njxo2699571aVLF2VnZzf4OoQ/AAAmh97q9/TTT2vSpEm68847tWPHDqWmpur222/X5MmTJf0wBSgrK9OcOXO0b98+paam6qqrrtK0adPCeq+/J/Djjw1yUPPMu5wuAYg6e1fNcroEICrFN3LrmvCrFyN2rqr5/xGxc0UKnT8AAAbbP9uf8AcAwGB7+PMhPwAAuAydPwAABts7f8IfAACD7eHP2B8AAJeh8wcAwGR340/4AwBgYuwPAACsQucPAIDB9s6f8AcAwED4AwDgNnZnP/f8AQBwGzp/AAAMjP0BAHAZ28OfsT8AAC5D5w8AgMH2zp/wBwDAYHv4M/YHAMBl6PwBADDZ3fgT/gAAmBj7AwAAq9D5AwBgsL3zJ/wBADAQ/gAAuI3d2c89fwAA3IbOHwAAA2N/WCcmxqMH7rhav7y6r5LbttL2nVV68Y0VmvH8W8Fjktqcpof/c6iyBnRXQsvmWvrxBuU/+oo2lu90sHKgcVVWVmrmE4/pHx98oEOHDqpDxzP10MOPKKNHT0lSyTt/1ysL/6AvP/9cVVX79PIfF6lb9+4OV43GQPjDOnePulK3XneJbp38or7YuF29Mzrqtw/epOr9B/WbBUskSQufvE1HjtZqxPjfqrrmkMbddIX+9uxYZV77sA4cOuzwKwAir7qqSqNu+qX69OuvZ559Xq3btFb51q1q1SoheMzBgweUmXm+srNzNHXKAw5WC5wcwt+FLjivs/6ypExvLf1cklS+fY+uH9RHfTLOlCR16Zik/uem6fzhD+vLTRWSpHGPvKwtJY/o+pze+v1ryxyrHWgsv3vheSX7fJo2vTC474wzOoQcM+SaYZKk7777tilLgwNs7/xZ8OdCy9du0uX9zlaXjkmSpJ5d22tAr876+z++kCR5m/3wN+Ghw0eDzwkEAjp8+Kgu7JXe9AUDTWDJe+8qI6OH7pkwTpddMkDXDx+mP72y0Omy4BCPxxOxLRqF3fnv2rVLv/vd77Rs2TJVVPzQFfp8Pl144YUaNWqU2rVrF/EiEVmPF7+jVi3jtfa1B1RbG1BsrEdTnvmL/vDmR5Kkr7dUqHz7Hk0be43ueniBag4e1ribLtcZvtbyJSac4OzAv6Zvv/1GC19eoP/IvVm33HaHPv/0UxUVPqy4uDhdM+zfnS4PiKiwwn/VqlXKzs5WixYtlJWVpa5du0r6YZHMU089pRkzZujtt99Wnz59jnsev98vv98fsi9QVytPTGyY5ePnuO6q8zUyp69G/XqOvti4Xeee3V6P3XOdtu+s0rw3Vujo0TqNvPt5zZ5yo7a//5iOHq3Vuyu+1ltLP1eU/hELnLS6uoAyevTQuPH5kqTu3c/Rhg3r9crCPxD+bmT5/9eFFf5jx47ViBEj9Oyzz9YbZQQCAd1xxx0aO3asli07/j3hwsJCTZ06NWRfbHJfxaX0C6cc/EyPjB+mx4vf0Stvr5Ykfb5hmzqmtNHEm6/UvDdWSJI++fIbXTByhlq1jFezuFO0a+9+vT/3Hq3+otzJ0oFG065dO3VOD72t1blzZ5W887ZDFcFJ0Tquj5Sw7vmvXbtWEyZMOOY/isfj0YQJE7RmzZoTnqegoEBVVVUh2ynJvcMpBSeheXwz1QXqQvbV1gUUE1P/fw7V+w9p1979Su/YTuef01F/WVzWVGUCTapX5vnasnlzyL6tW7YoNbW9QxUBjSeszt/n82nlypXq1q3bMR9fuXKlkpOTT3ger9crr9cbso+Rf9P52/uf6r5bsvXN9r36YuN29ep2hsbddLnmLloePObarEzt3Ltf31TsUY+zUvX4xOv0xuIylS7/ysHKgcZz0//JVe5Nv9R/P/esrsrO0WeflumPf1yoyQ8+FDymat8+bd++XTt37pAkbdnywx8LiYmJSmS9k1Vs7/zDCv977rlHt912m1avXq2BAwcGg76yslKlpaV6/vnn9fjjjzdKoYic/KJXNOXOf9N//foGtWvdUtt3VumFP/5Djzz3ZvAYX7tWKrr7WiW1PU0Vu6o17y8rVPjcW8c5K/CvrUfPc/XEf83SUzOf0G9nP6P2Z5yhe+/7tQb/2zXBYxa/964mP1AQ/Pm+eyZIku648y6NHjO2yWtG47E8++UJBAKBcJ7w8ssv68knn9Tq1atVW1srSYqNjVXv3r2Vn5+v66+//mcV0jzzrp/1PMBme1fNcroEICrFN/Kn1Jw1MXLNzvrHBkXsXJES9j/fDTfcoBtuuEFHjhzRrl27JP0w8oqLi4t4cQAAIPJ+9t9OcXFxSklJiWQtAABEBdvH/ny8LwAABtsX/PHxvgAAuAydPwAABssbf8IfAABTTIzd6c/YHwAAl6HzBwDAwNgfAACXYbU/AACwCp0/AAAGyxt/wh8AAJPtY3/CHwAAg+3hzz1/AABchs4fAACD5Y0/4Q8AgImxPwAAsArhDwCAweOJ3BaO2tpaTZo0SWlpaWrevLnS09M1bdo0BQKB4DGBQECTJ09WSkqKmjdvrqysLK1fvz6s6xD+AAAYPB5PxLZwFBUVafbs2Zo1a5a+/PJLFRUV6dFHH9XTTz8dPObRRx/VU089pWeffVYrVqzQqaeequzsbB06dKjB1+GePwAAUeLDDz/U0KFDNXjwYElSp06dtGDBAq1cuVLSD13/zJkz9cADD2jo0KGSpLlz5yo5OVmLFi3SyJEjG3QdOn8AAAyRHPv7/X5VV1eHbH6//5jXvfDCC1VaWqp169ZJktauXaulS5cqJydHkrR582ZVVFQoKysr+JyEhAT1799fy5Yta/DrI/wBADBEcuxfWFiohISEkK2wsPCY173//vs1cuRIdevWTXFxccrMzNT48eN14403SpIqKiokScnJySHPS05ODj7WEIz9AQBoRAUFBcrPzw/Z5/V6j3nswoULNW/ePM2fP18ZGRlas2aNxo8fr9TUVOXm5kasJsIfAABDJN/m7/V6fzLsTRMnTgx2/5LUs2dPbd26VYWFhcrNzZXP55MkVVZWKiUlJfi8yspK9erVq8E1MfYHAMDg1Gr/AwcOKCYmNJpjY2NVV1cnSUpLS5PP51NpaWnw8erqaq1YsUIDBgxo8HXo/AEAMDj1AX9DhgzR9OnT1bFjR2VkZOiTTz7RE088oby8vP+py6Px48fr4Ycf1llnnaW0tDRNmjRJqampGjZsWIOvQ/gDABAlnn76aU2aNEl33nmnduzYodTUVN1+++2aPHly8Jh7771XNTU1uu2227Rv3z5dfPHFeuuttxQfH9/g63gCP/7YIAc1z7zL6RKAqLN31SynSwCiUnwjt64Dit6P2LmW3feLiJ0rUuj8AQAwWP69Piz4AwDAbej8AQAw2P6VvoQ/AAAGy7OfsT8AAG5D5w8AgIGxPwAALmN7+DP2BwDAZej8AQAwWN74E/4AAJhsH/sT/gAAGCzPfu75AwDgNnT+AAAYGPsDAOAylmc/Y38AANyGzh8AAEOM5a0/4Q8AgMHy7GfsDwCA29D5AwBgYLU/AAAuE2N39hP+AACYbO/8uecPAIDL0PkDAGCwvPEn/AEAMHlkd/oz9gcAwGXo/AEAMLDaHwAAl2G1PwAAsAqdPwAABssbf8IfAACT7d/qx9gfAACXofMHAMBgeeNP+AMAYLJ9tT/hDwCAwfLs554/AABuQ+cPAIDB9tX+hD8AAAa7o5+xPwAArkPnDwCAgdX+AAC4jO3f6sfYHwAAl6HzBwDAwNgfAACXsTz7GfsDAOA2dP4AABgY+wMA4DK2r/Yn/AEAMNje+XPPHwAAl6HzBwDAYHffT/gDAFCP7d/qx9gfAACXofMHAMBgeeNP+AMAYGK1PwAAsAqdPwAABssbfzp/AABMMR5PxLZwdOrUSR6Pp942ZswYSdJll11W77E77rgj7NdH5w8AQJRYtWqVamtrgz9/9tlnuvLKKzVixIjgvltvvVUPPfRQ8OcWLVqEfR3CHwAAg1Nj/3bt2oX8PGPGDKWnp+vSSy8N7mvRooV8Pt9JXYexPwAAhmON3n/u5vf7VV1dHbL5/f4T1nD48GG99NJLysvLC3n3wbx585SYmKgePXqooKBABw4cCP/1BQKBQNjPagSjFpQ5XQIQddqfHu90CUBUmp7TtVHPP/a1LyN2rrZrX9bUqVND9k2ZMkUPPvjgcZ+3cOFC/epXv1J5eblSU1MlSc8995zOPPNMpaamqqysTPfdd5/69eunV199NayaCH8gihH+wLH9K4X/41d3rtfpe71eeb3e4z4vOztbzZo10xtvvPGTx7z77rsaOHCgNmzYoPT09AbXxD1/AAAMkfyQn4YEvWnr1q0qKSk5YUffv39/SSL8AQA4WTEOv8+/uLhYSUlJGjx48HGPW7NmjSQpJSUlrPMT/gAARJG6ujoVFxcrNzdXp5zyz5jeuHGj5s+fr6uvvlpt27ZVWVmZJkyYoF/84hc699xzw7oG4Q8AgMHJzr+kpETl5eXKy8sL2d+sWTOVlJRo5syZqqmpUYcOHTR8+HA98MADYV+D8AcAwODkF/tcddVVOtZa/A4dOmjJkiURuQbv8wcAwGXo/AEAMDi94K+xEf4AABj4Vj8AAGAVOn8AAAzhfhXvvxrCHwAAg+1jccIfAACD5Y2/9X/cAAAAA50/AAAG7vkDAOAylmc/Y38AANyGzh8AAAOf8AcAgMvYfs+fsT8AAC5D5w8AgMHyxp/wBwDAZPs9f8b+AAC4DJ0/AAAGj+xu/Ql/AAAMto/9CX8AAAy2hz/3/AEAcBk6fwAADB7L3+tH+AMAYGDsDwAArELnDwCAwfKpP+EPAICJL/YBAABWofMHAMBg+4I/wh8AAIPlU3/G/gAAuA2dPwAAhhi+2AcAAHexfexP+AMAYLB9wR/3/AEAcBk6fwAADLZ/yA/hDwCAwfLsZ+wPAIDb0PkDAGBg7A8AgMtYnv2M/QEAcBs6fwAADLZ3xoQ/AAAGj+Vzf9v/uAEAAAY6fwAADHb3/YQ/AAD18FY/AABcxu7o554/AACuQ+cPAIDB8qk/4Q8AgIm3+gEAAKvQ+QMAYLC9Myb8AQAwMPYHAABWofMHAMBgd99P+AMAUA9jfwAA0CQ6deokj8dTbxszZowk6dChQxozZozatm2rli1bavjw4aqsrAz7OoQ/AACGmAhu4Vi1apW2b98e3N555x1J0ogRIyRJEyZM0BtvvKFXXnlFS5Ys0bZt23TttdeG/foY+wMAYHBq7N+uXbuQn2fMmKH09HRdeumlqqqq0gsvvKD58+friiuukCQVFxere/fuWr58uS644IIGX4fOHwAAgyeCm9/vV3V1dcjm9/tPWMPhw4f10ksvKS8vTx6PR6tXr9aRI0eUlZUVPKZbt27q2LGjli1bFtbrI/wBAGhEhYWFSkhICNkKCwtP+LxFixZp3759GjVqlCSpoqJCzZo10+mnnx5yXHJysioqKsKqibE/AACGSE79CwoKlJ+fH7LP6/We8HkvvPCCcnJylJqaGrli/gfhDwCAISaC7/T3er0NCvsf27p1q0pKSvTqq68G9/l8Ph0+fFj79u0L6f4rKyvl8/nCOj9jfwAAokxxcbGSkpI0ePDg4L7evXsrLi5OpaWlwX1ff/21ysvLNWDAgLDOT+cPAIDByc/4qaurU3FxsXJzc3XKKf+M6YSEBN1yyy3Kz89XmzZt1KpVK40dO1YDBgwIa6W/RPgDAFCPx8EP+C0pKVF5ebny8vLqPfbkk08qJiZGw4cPl9/vV3Z2tn7zm9+EfQ1PIBAIRKLYkzVqQZnTJQBRp/3p8U6XAESl6TldG/X8f/1sR8TONbhHUsTOFSl0/gAAGCz/aH/CHwAAUyRX+0cjVvsDAOAydP4AABgY+wMA4DKEPwAALuPkW/2aAvf8AQBwGTp/AAAMMXY3/oQ/AAAmxv4AAMAqdP4AABhY7Q8AgMsw9gcAAFah8wcAwMBqf1jn8SHdlNiyWb39pet26cXV2yRJ6W1baPh5PqW3baG6QEDlew/q8cWbdaQ2Kr4BGmgUf5t6iw7srf9VrukXX63M60YHfw4EAlr62wdV+dXHGpD3a7U/d0BTlokmYPvYn/B3oal/X6+YH61maZ8Qr3uv6KxV31RJ+iH4774sTX/9Yode+ug71QWkDqfHK0Duw3ID735Cgbq64M9V27fqg9mT1P68i0OOW7/kz/LYviIMVuOevwt9769V1aGjwa1X+9NU+b1fX+2okST96vwUlazbpb9+uVPbqv2q+N6vVd9U6Wgd6Q+7eVsmKL5V6+C2/fNVOjUxRe269Ages+/bTVr/3iL1+eV/OlgpGpvHE7ktGtH5u1xsjEcDOrXW21/tlCSd5o1VeuKpWrZln/5vVrqSTmum7dV+/WlthdbvOuBwtUDTqTt6ROWr39NZlw0LdvlHDx/SihcfV+Z1dyi+VWuHK0RjitLMjhg6f5c7v30rtYiL1dLNeyVJSS29kqRhPZO1ZOMe/b/Fm7V1z0Hde0VnJR9jnQBgq+8+Xa4jB2vUqd/A4L61r/232qZ1U2rPCxysDE0hxuOJ2BaNIh7+33zzjfLy8o57jN/vV3V1dchWe+RwpEtBA/wivY0+3f699h08KumfI6r3NuzR0s17Vb73kBZ8sl0V3/t1SXobBysFmtaW5e/I1723mie0lSRt+2yFdq4vU69/v9XhyoCTF/Hw37Nnj+bMmXPcYwoLC5WQkBCyffrnFyJdCk6gbYs4ZSS31JKNe4L79h08IknaVn0o5NhtVX61bRHXpPUBTqnZs0OV69Yq7YKrgvt2rCvT/t0V+nPBSP0pf6j+lD9UkrSseIYWP13gVKloJJ4IbtEo7Hv+r7/++nEf37Rp0wnPUVBQoPz8/JB9YxatC7cUnKRLOrdRtf+o1m6rDu7bVXNEew8cUcpp3pBjfa28Ktv2fVOXCDhiy4oSxZ+WIN85fYP7umVdp7QBV4Uc907RXTpv2C1K7dGvqUtEY4vW1I6QsMN/2LAfFr8EjvO+rxO9Bcbr9crrDQ2X2DjuJzclj6SLO7fWPzbvlbmI/82vdmpYj2SV7zuk8r0HdXFaa6Wc5tWsTVsdqRVoSoG6Om1dWaIz+16hmNjY4P7/fQeAqUXrdjq1ra8pSwROWthj/5SUFL366quqq6s75vbxxx83Rp2IsHN8LZV4ajO9v2lPvcf+/vUu/fWLHfplZoqm5XRV9+SWeuy9Tdq5n3UZsF/lujU6sHenOvW/0ulS4CBPBP+LRp7A8Vr4Y7jmmmvUq1cvPfTQQ8d8fO3atcrMzFTdjz4ooyFGLSgL63jADdqfHu90CUBUmp7TtVHPv3JTVcTO1a9zQsTOFSlhj/0nTpyompqan3y8S5cueu+9906qKAAA0HjCDv9LLrnkuI+feuqpuvTSS392QQAAOC06h/WRwyf8AQBgsjz9+YQ/AABchs4fAABDtK7SjxTCHwAAQ5R+JH/EEP4AABgsz37u+QMA4DZ0/gAAmCxv/Ql/AAAMti/4Y+wPAIDL0PkDAGBgtT8AAC5jefYz9gcAwG3o/AEAMFne+hP+AAAYWO0PAACsQucPAICB1f4AALiM5dlP+AMAUI/l6c89fwAAXIbOHwAAg+2r/Ql/AAAMti/4Y+wPAIDL0PkDAGCwvPEn/AEAqMfy9GfsDwCAy9D5AwBgYLU/AAAuw2p/AABgFcIfAACDJ4JbuL777jvddNNNatu2rZo3b66ePXvqo48+Cj4+atQoeTyekG3QoEFhXYOxPwAAJofG/nv37tVFF12kyy+/XG+++abatWun9evXq3Xr1iHHDRo0SMXFxcGfvV5vWNch/AEAMDi14K+oqEgdOnQICfa0tLR6x3m9Xvl8vp99Hcb+AAA0Ir/fr+rq6pDN7/cf89jXX39dffr00YgRI5SUlKTMzEw9//zz9Y5bvHixkpKSdPbZZ2v06NHavXt3WDUR/gAAGDyeyG2FhYVKSEgI2QoLC4953U2bNmn27Nk666yz9Pbbb2v06NEaN26c5syZEzxm0KBBmjt3rkpLS1VUVKQlS5YoJydHtbW1DX99gUAgcNL/ShEwakGZ0yUAUaf96fFOlwBEpek5XRv1/Bt3HIzYuc5IiKnX6Xu93mPep2/WrJn69OmjDz/8MLhv3LhxWrVqlZYtW3bM82/atEnp6ekqKSnRwIEDG1QTnT8AAI3I6/WqVatWIdtPLdBLSUnROeecE7Kve/fuKi8v/8nzd+7cWYmJidqwYUODa2LBHwAAJodW+1900UX6+uuvQ/atW7dOZ5555k8+59tvv9Xu3buVkpLS4OvQ+QMAYPBE8L9wTJgwQcuXL9cjjzyiDRs2aP78+Xruuec0ZswYSdL+/fs1ceJELV++XFu2bFFpaamGDh2qLl26KDs7u8HXIfwBAIgSffv21WuvvaYFCxaoR48emjZtmmbOnKkbb7xRkhQbG6uysjJdc8016tq1q2655Rb17t1bH3zwQVjv9WfBHxDFWPAHHFtjL/jbvOtQxM6Vlhh9v8fc8wcAwGD59/ow9gcAwG3o/AEAMFne+hP+AAAYnPps/6ZC+AMAYPDYnf3c8wcAwG3o/AEAMFje+BP+AACYGPsDAACr0PkDAFCP3a0/4Q8AgIGxPwAAsAqdPwAABssbf8IfAAATY38AAGAVOn8AAAx8tj8AAG5jd/YT/gAAmCzPfu75AwDgNnT+AAAYbF/tT/gDAGCwfcEfY38AAFyGzh8AAJPdjT/hDwCAyfLsZ+wPAIDb0PkDAGBgtT8AAC7Dan8AAGAVOn8AAAy2j/3p/AEAcBk6fwAADHT+AADAKnT+AAAYbF/tT/gDAGBg7A8AAKxC5w8AgMHyxp/wBwCgHsvTn7E/AAAuQ+cPAICB1f4AALgMq/0BAIBV6PwBADBY3vgT/gAA1GN5+hP+AAAYbF/wxz1/AABchs4fAACD7av9PYFAIOB0EYgefr9fhYWFKigokNfrdbocICrwewHbEP4IUV1drYSEBFVVValVq1ZOlwNEBX4vYBvu+QMA4DKEPwAALkP4AwDgMoQ/Qni9Xk2ZMoVFTcCP8HsB27DgDwAAl6HzBwDAZQh/AABchvAHAMBlCH8AAFyG8EfQM888o06dOik+Pl79+/fXypUrnS4JcNT777+vIUOGKDU1VR6PR4sWLXK6JCAiCH9Ikl5++WXl5+drypQp+vjjj3XeeecpOztbO3bscLo0wDE1NTU677zz9MwzzzhdChBRvNUPkqT+/furb9++mjVrliSprq5OHTp00NixY3X//fc7XB3gPI/Ho9dee03Dhg1zuhTgpNH5Q4cPH9bq1auVlZUV3BcTE6OsrCwtW7bMwcoAAI2B8Id27dql2tpaJScnh+xPTk5WRUWFQ1UBABoL4Q8AgMsQ/lBiYqJiY2NVWVkZsr+yslI+n8+hqgAAjYXwh5o1a6bevXurtLQ0uK+urk6lpaUaMGCAg5UBABrDKU4XgOiQn5+v3Nxc9enTR/369dPMmTNVU1Ojm2++2enSAMfs379fGzZsCP68efNmrVmzRm3atFHHjh0drAw4ObzVD0GzZs3SY489poqKCvXq1UtPPfWU+vfv73RZgGMWL16syy+/vN7+3Nxc/f73v2/6goAIIfwBAHAZ7vkDAOAyhD8AAC5D+AMA4DKEPwAALkP4AwDgMoQ/AAAuQ/gDAOAyhD8AAC5D+AMA4DKEPwAALkP4AwDgMoQ/AAAu8/8BuVuA6gvqUzYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# confusion matrix of the binary model\n",
    "with open(\"models/svc.pkl\", \"rb\") as f:\n",
    "    svc = pickle.load(f)\n",
    "\n",
    "X_test = values_test.drop([\"path\", \"correct\", \"predicted\", \"true\"], axis=1)\n",
    "y_true = values_test[\"correct\"]\n",
    "y_pred = svc.predict(X_test)\n",
    "print(\"Accuracy:\", accuracy_score(y_true, y_pred))\n",
    "print(\"Precision:\", precision_score(y_true, y_pred, average=\"macro\"))\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [2:01:21<00:00, 24.27s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf8AAAGiCAYAAADp4c+XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5DElEQVR4nO3de1xUdf7H8fegMojAKKBcFNS8kXcjI0rNW5q1puV2U0vNai10U7YydivTLri1lbVecs20NsnNdrW1Lc1MMX9pKUWZd8hrAt4CBGFQ4PeH7mxzMHVq9ExnXs99nMfD+Z7DOR8W7c3ne75zxlZdXV0tAADgNwLMLgAAAFxchD8AAH6G8AcAwM8Q/gAA+BnCHwAAP0P4AwDgZwh/AAD8DOEPAICfIfwBAPAzhD8AAH6G8AcAwAdNnTpVNptN48ePd4317NlTNpvNbRszZozH567txToBAIAXbNiwQbNnz1bHjh1r7Lv33ns1ZcoU1+vg4GCPz0/nDwCADykpKdGwYcM0Z84cNWjQoMb+4OBgRUdHu7awsDCPr0H4AwBwATmdThUXF7ttTqfzJ49PSUnRDTfcoL59+55x/4IFCxQZGan27dsrLS1Nx48f97gmn5n2n/P5HrNLwGkfbT1idgk4bdK1rc0uAafN+mKf2SXgR2bcdOkFPX/dLmO9dq6JgyI1efJkt7FJkybpySefrHHswoUL9eWXX2rDhg1nPNfQoUPVtGlTxcbG6ptvvtHEiRO1fft2/etf//KoJp8JfwAAfIbNexPjaWlpSk1NdRuz2+01jtu3b58efPBBrVixQkFBQWc813333ef6c4cOHRQTE6M+ffooNzdXLVq0OO+aCH8AAC4gu91+xrA3ysrK0sGDB3XZZZe5xiorK7VmzRpNnz5dTqdTtWrVcvuapKQkSVJOTg7hDwDAL2KzXfRL9unTR5s2bXIbGzVqlBISEjRx4sQawS9J2dnZkqSYmBiPrkX4AwBg5MVp//MVGhqq9u3bu43Vq1dPERERat++vXJzc5WRkaHrr79eERER+uabbzRhwgT16NHjjG8JPBvCHwAAIxM6/3MJDAzUxx9/rGnTpqm0tFRxcXEaMmSIHnvsMY/PRfgDAOCjVq9e7fpzXFycMjMzvXJewh8AACMTpv0vJsIfAAAjH5z29yZr/2oDAABqoPMHAMCIaX8AAPwM0/4AAMBK6PwBADBi2h8AAD/DtD8AALASOn8AAIyY9gcAwM9YfNqf8AcAwMjinb+1vzsAAFADnT8AAEYW7/wJfwAAjAKsfc/f2r/aAACAGuj8AQAwYtofAAA/Y/G3+ln7VxsAAFADnT8AAEZM+wMA4GeY9gcAAFZC5w8AgBHT/gAA+BmLT/sT/gAAGFm887f2dwcAAGqg8wcAwIhpfwAA/AzT/gAAwEro/AEAMGLaHwAAP8O0PwAAuNimTp0qm82m8ePHu8bKy8uVkpKiiIgIhYSEaMiQISooKPD43IQ/AABGtgDvbT/Dhg0bNHv2bHXs2NFtfMKECVq6dKkWLVqkzMxMHThwQDfffLPH5yf8AQAwstm8t3mopKREw4YN05w5c9SgQQPXeFFRkebOnasXX3xRvXv3VmJioubNm6fPPvtM69ev9+ga3PP30L5t32jDB4tUsHunSguPatCDk9Qq8WrX/oryMq15Z65ysj5TeUmxwhpG67J+g9W5929MrNqaBneIUlLT+mrsCFLFySptP1SqBRu/14Fip9txrRvW0x2XxaplZLCqqqXdR4/rmRU5qqisNqly/1B2vFRvvz5Ln69dpeLCH9S8ZRvdPfYhtUxoZ3ZpltavdYQ6x4YqKiRQJ6qq9d2RMi3ZfFAHSypcx9zROVptGtaTo25tOU9WadfRMi359qAKfnQMvMfpdMrpdP/vkt1ul91uP+PxKSkpuuGGG9S3b189/fTTrvGsrCydOHFCffv2dY0lJCQoPj5e69at05VXXnneNRH+HjrhLFej+EvUoUd/vffKlBr7V2e8qr1bvtb1YybKERml3d9m6eM3/qqQ+hFqeVmyCRVbV7voEC3fdkg5h4+rls2moZfF6rF+LTVhyVY5T1ZJOhX8f7q2pRZvytfcz/epqqpaTcPrqorcv+Bm/uUp7d2Vq9+nPaXwyIZas+IDTX74fk17/V1FNGxkdnmW1SoyWGu++0F7fihTgM2mG9s10rir4/XUx7muX3j3FpZrw74iHS07qXp1aun6SyM19up4PbE8R/zTOM2LC/7S09M1efJkt7FJkybpySefrHHswoUL9eWXX2rDhg019uXn5yswMFD169d3G4+KilJ+fr5HNTHt76FLOl2hbr8dpVaXdzvj/u93blG7bn0Vf2knORpGq1OvG9Qo/hLlf7ftIldqfc+syNXqnKPaX1iuPT+UacbaPWoYYtclEcGuY0Zc0UQfbD2oJZsKtL+wXAeKnVq3u1AnSf8Lyuks1/o1n+iu3/1e7TpdppjGcbpt5O8UHRun5f9+1+zyLG3GZ/u0fm+R8o5V6Ptip/6edUDhwXUUXz/Idcz/7S5UzpEyHT1+QvuKyrV0yyGFB9dRRL06JlbuY7w47Z+WlqaioiK3LS0trcYl9+3bpwcffFALFixQUFDQGYryHo87/8OHD+v111/XunXrXL9pREdH66qrrtLIkSPVsGFDrxf5a9K4VVvlfLVe7Xtcp5AGEdq39Wsdzf9ePYeOMbs0ywsOrCVJKnGelCSFBdVW64b19GnuUT19fWtFhdp1oKhcb395QNsOlppZquVVVVaqqqpSdQLdpzUD7XZt+zbbnKL8VN06p3q80oqqM+4PrGVTclOHDpdW6IfjJy5mab7Ni53/2ab4fywrK0sHDx7UZZdd5hqrrKzUmjVrNH36dC1fvlwVFRUqLCx06/4LCgoUHR3tUU0ehf+GDRvUv39/BQcHq2/fvmrdurXrwq+88oqmTp2q5cuX6/LLLz/rec50/+NEhbPGfyh+jXrfmaKPXp+m2eOHKqBWLdlsAep393jFJXQ89xfjZ7NJGnlFE20rKNG+wnJJUlRooCTp1s4xenPjfu0+WqZrWoTrif6tlLpkq/KPOc9yRvwSdYPrqU3bjnr376+pSXxzORqEa+0ny7VjyyZFx8aZXZ7fsEka0jFKuUeOK8/w97178wa6qX0j2WsHKP+YU3/9v71iGYy5+vTpo02bNrmNjRo1SgkJCZo4caLi4uJUp04drVy5UkOGDJEkbd++XXv37lVysme3lT0K/3HjxumWW27Rq6++KpthBWN1dbXGjBmjcePGad26dWc9z5nuf/xm9IO68d4JnpTjk75a8Z7ycrfppgmTFRYRpX3bN+njN6crpH6Emra/7NwnwM9yz5VximsQpMc/2OEas+nU39EVOw5rdc5RSdLuo9+rQ0yYereKUMaXB0yp1V/8Pm2KZjw/Rffeep0CAmrpklYJ6ta7v3J3bDW7NL9xW6doxYba9eKaPTX2bdhXpG0HS+QIqq0+rSI0umtjvbBmD7fE/suEJ/yFhoaqffv2bmP16tVTRESEa3z06NFKTU1VeHi4wsLCNG7cOCUnJ3u02E/yMPy//vprzZ8/v0bwS5LNZtOECRPUpUuXc54nLS1NqampbmNvfe3ZYgVfdKLCqU8XzdOgByepReckSVLD+Et0aG+uNnz4LuF/gYxOaqLL4hya9OEOHf3RtGVh2ak/7z89E/Bf3xeVK7Je4EWt0R9FN47TU9PmqLysTGXHS9QgoqFemPKoomIam12aX7i1Y5TaR4fopU/3qLD8ZI395SerVH6ySodKT2jX0f16/jdt1Ck2VFn7i02o1vecKed8wUsvvaSAgAANGTJETqdT/fv318yZMz0+j0fhHx0drS+++EIJCQln3P/FF18oKirqnOc50/2POoE/eFKKT6qqPKmqypM1/tLYAgJUXX3m+234ZUYnNdEV8fU1adlOt7cySdLBkgodLa1QrMP971pMmF1ffc9/4C6WoLp1FVS3rkqOFSt7wzrd+bsHzS7J8m7tGKVOsaGa9ukeHTmP+/g226l5sjoBvhl4/mz16tVur4OCgjRjxgzNmDHjF53Xo/B/6KGHdN999ykrK0t9+vRxBX1BQYFWrlypOXPm6C9/+csvKsjXVZSXqbDgf9PFRYfydXBProLqhSosspGaJHRU5sI5qh1oV1hkI+3ftklb1n6snkN/Z2LV1nTPlXHqdkkDPbfyO5WfrFT9uqf+Oh+vqHS9pem9zQW6rXOs9hwtO3XPv2W4GjuC9MLq78ws3S98teEzqVqKjWuq/O/36c3ZL6txfDP1vm6g2aVZ2m2donV5kzDNXr9fzpNVCrOfWghbdqJKJ6qqFRFcR4lNwrT1YKlKnCdVv24d9WsdoYqqKn2bX2Jy9b7DVzt/b/Eo/FNSUhQZGamXXnpJM2fOVGVlpSSpVq1aSkxM1Pz583XrrbdekEJ9Rf6uHXon/WHX69UZsyVJ7bpdqwH3PayBD/xRaxa9rg9enarykmMKi2ykbr8dqU485Mfr+iecemfJ5AGt3cZnrN3tusf/wZZDCqwVoBFXNFFIYC3t+aFMT320UwXHeJjJhXa8tEQL5kzXkcMHFRIapiu799HQ0Q+odm3eTnYh9bjk1BPhJvRo6jb+96wDWr+3SCerqtUyIli9WoQrOLCWjpWfVM6R43ohc49KKirNKNk3WTv7Zauurv5ZqztOnDihw4cPS5IiIyNVp84v+wc95/OaC1Jgjo+2HjG7BJw26drW5z4IF8WsL/aZXQJ+ZMZNl17Q89e7ZZ7XzlW6aJTXzuUtP/sJf3Xq1FFMTIw3awEAwCcw7Q8AgJ+xevjzeF8AAPwMnT8AAAZW7/wJfwAADAh/AAD8jbWzn3v+AAD4Gzp/AAAMmPYHAMDPWD38mfYHAMDP0PkDAGBg9c6f8AcAwMDq4c+0PwAAfobOHwAAI2s3/oQ/AABGTPsDAABLofMHAMDA6p0/4Q8AgAHhDwCAv7F29nPPHwAAf0PnDwCAAdP+AAD4GauHP9P+AAD4GTp/AAAMrN75E/4AABhYPfyZ9gcAwM/Q+QMAYGTtxp/wBwDAiGl/AABwUcyaNUsdO3ZUWFiYwsLClJycrA8//NC1v2fPnrLZbG7bmDFjPL4OnT8AAAZmdf5NmjTR1KlT1apVK1VXV+uNN97QoEGD9NVXX6ldu3aSpHvvvVdTpkxxfU1wcLDH1yH8AQAwMCv8Bw4c6Pb6mWee0axZs7R+/XpX+AcHBys6OvoXXYdpfwAAjGze25xOp4qLi902p9N5zhIqKyu1cOFClZaWKjk52TW+YMECRUZGqn379kpLS9Px48c9/vYIfwAALqD09HQ5HA63LT09/SeP37Rpk0JCQmS32zVmzBgtXrxYbdu2lSQNHTpUb731llatWqW0tDT9/e9/1/Dhwz2uiWl/AAAMvDntn5aWptTUVLcxu93+k8e3adNG2dnZKioq0rvvvqsRI0YoMzNTbdu21X333ec6rkOHDoqJiVGfPn2Um5urFi1anHdNhD8AAAbeDH+73X7WsDcKDAxUy5YtJUmJiYnasGGDXn75Zc2ePbvGsUlJSZKknJwcj8KfaX8AAHxYVVXVT64RyM7OliTFxMR4dE46fwAADMxa7Z+WlqYBAwYoPj5ex44dU0ZGhlavXq3ly5crNzdXGRkZuv766xUREaFvvvlGEyZMUI8ePdSxY0ePrkP4AwBgYFb4Hzx4UHfddZfy8vLkcDjUsWNHLV++XNdee6327dunjz/+WNOmTVNpaani4uI0ZMgQPfbYYx5fh/AHAMBHzJ079yf3xcXFKTMz0yvXIfwBADCy9qP9fSf8HfY6ZpeA0z54/2uzS8BpwxNjzS4Bp70w8FKzS8BFxAf7AAAAS/GZzh8AAF9h9c6f8AcAwMDi2U/4AwBgZPXOn3v+AAD4GTp/AAAMLN74E/4AABgx7Q8AACyFzh8AAAOLN/6EPwAARgEB1k5/pv0BAPAzdP4AABgw7Q8AgJ9htT8AALAUOn8AAAws3vgT/gAAGFl92p/wBwDAwOrhzz1/AAD8DJ0/AAAGFm/8CX8AAIyY9gcAAJZC5w8AgIHFG3/CHwAAI6b9AQCApdD5AwBgYPHGn/AHAMCIaX8AAGApdP4AABhYvPEn/AEAMGLaHwAAP2OzeW/zxKxZs9SxY0eFhYUpLCxMycnJ+vDDD137y8vLlZKSooiICIWEhGjIkCEqKCjw+Psj/AEA8BFNmjTR1KlTlZWVpY0bN6p3794aNGiQNm/eLEmaMGGCli5dqkWLFikzM1MHDhzQzTff7PF1mPYHAMDArGn/gQMHur1+5plnNGvWLK1fv15NmjTR3LlzlZGRod69e0uS5s2bp0svvVTr16/XlVdeed7XIfwBADDwZvY7nU45nU63MbvdLrvdftavq6ys1KJFi1RaWqrk5GRlZWXpxIkT6tu3r+uYhIQExcfHa926dR6FP9P+AABcQOnp6XI4HG5benr6Tx6/adMmhYSEyG63a8yYMVq8eLHatm2r/Px8BQYGqn79+m7HR0VFKT8/36Oa6PwBADDw5rR/WlqaUlNT3cbO1vW3adNG2dnZKioq0rvvvqsRI0YoMzPTa/VIhD8AADV4c9r/fKb4fywwMFAtW7aUJCUmJmrDhg16+eWXddttt6miokKFhYVu3X9BQYGio6M9qolpfwAAfFhVVZWcTqcSExNVp04drVy50rVv+/bt2rt3r5KTkz06J50/AAAGZq32T0tL04ABAxQfH69jx44pIyNDq1ev1vLly+VwODR69GilpqYqPDxcYWFhGjdunJKTkz1a7CcR/gAA1GBW+B88eFB33XWX8vLy5HA41LFjRy1fvlzXXnutJOmll15SQECAhgwZIqfTqf79+2vmzJkeX4fwBwDAR8ydO/es+4OCgjRjxgzNmDHjF12H8PfQ7i1fa+3Sf+jArh069sMR3fHQU2rbtZvbMQf379FHGX/T7i1fq6qqUo0aN9Xtf5is+pFRJlVtTff0a6N7+ycovmGIJGnrvkJNfTdbH331vSSpeVSonr2rq5ITomSvE6AV2d/robnrdbCo3MyyLem7LV8r8723tf+7U/8u7nrkabW/ortr/z+mpytr9TK3r2nd+Qrd89jzF7tUv5S1cYPmvz5XW7d8q0OHDumlV2aod5++5/5CP2bxR/sT/p6qcJYrumkLXdZrgN5+4Yka+4/mf6/XJv1eib0GqPctIxVUN1gF+3erdp1AE6q1tu+PHNcTb2UpJ69YNps0rGdL/eORPrrq4X9rz6ES/fvxftq05wfdMPlU6Dx+excterSvev7xfVVXm1y8xVSUlymmWUt17X293nz+8TMe06bzFbo15VHX61r8m7hoysqOq02bNhp88xClPjjW7HJ+Faz+wT6Ev4dad0lS6y5JP7l/xcK5at0lSf2Hj3GNhUc3vhil+Z0Ps/a5vZ789pe6p1+CurZuqNiIYDVtGKKrHv63jpWdkCTdN/1TfT9/mHq2j9GqTXlmlGxZCZddqYTLzr7gqHadQIU2iLhIFeHHunW/Rt26X2N2Gb8qFs9+wt+bqqqqtOOr9ep24+1645mHlbc7Rw0aRav74GE1bg3AuwICbLo5uZnqBdXWFzsOqnlUmKolOU9Uuo4pr6hUVXW1ki+NIvxNkLs5W5PvHqS6IaFq2b6L+t9xj+qFOswuC/BLpoT/mZ5zfKLCqTqB5/8QBF9UWlyoivIyffre2+p7293qN+x32pn9hRa+8IRGPfGimrftbHaJltMuvoE+eeYGBQXWUkn5Cd3x3Cfatr9Ih4vLVVp+Uk8Pv1yTMrJks9k0ZViiatcKUHT9YLPL9jttOl+h9kk9FN4oWkcKDmhZxhy9/swjSnlmpgJq1TK7PKAGq0/7e/0hP/v27dPdd9991mPO9JzjJa9P93YpF111VZUkKeHyq3TVDbcopllL9Rg8VK0vS9aGFUtNrs6adhwoUvLD7+matPf12vLtmj22uxKaOHS42Kk7X1ylAZfH6eBbdyrvzWGqXy9QX+UeVhU3/C+6zt36qF3XqxXTtIXaX9Fdo9Kmal/ONuVuzja7NOCMbDbvbb7I6+F/9OhRvfHGG2c9Ji0tTUVFRW7b4Lt//YtQgsMcCqhVS40aN3Mbb9g4XkWHC8wpyuJOnKzSd/nHlP3dEU3KyNK3e47qgevbSZJWfn1AHcb+U81Gv634UW/rnr9+qtjwYO0uOGZy1YiIilW9MIeO5H9vdimAX/J42v/f//73Wfd/99135zzHmZ5zXCewxNNSfE7t2nXUuEWCDue5L0Q7krdfjoa8ze9iCLDZZK/j/jvtkWOnbjFd0z5GDR119Z+Ne80oDT9SeOSgjh8rZgEgfFaAr7bsXuJx+A8ePFg2m03VZ5k6tfK9Emd5mY7+qFspPJinvN05qhsSqvqRUeo28Da9M22Kml3aUc3bddHO7C+0Pesz3T1pmnlFW9TkoYn66Kv92ne4VKF16+jWbpeoe7toDXr6I0nSnb1auu7/J7VuqOfuTtL09zdr54Fikyu3HmfZcbcu/mhBng7s2qm6IWEKDgnVikVvqMOVPRRaP1xH8g/og7deVUR0Y7Xp3NXEqv3H8dJS7d37v196v9+/X9u2bpXD4VBMbKyJlfkuC8eYpJ8R/jExMZo5c6YGDRp0xv3Z2dlKTEz8xYX5qgO52/X6lAmu1x++eeqxil2u6a+bH3hUba/oroH3TtCaJRn6z7y/KjI2TrenTlbThA5mlWxZDR1BmjOuu6IbBKv4eIW+3fODBj39kT755oAkqVWsQ5OHJqpBiF17DpXo+X9+o7++v9nkqq1pf+52zX5yvOv1+2+cevpYYs/rdPO9qcrfk6us1ctUfrxEYQ0i1arT5ep/+2ief3GRbN78re4ZdZfr9V+eO/VZ8jcOuklPPTvVrLJgIlv12Vr4M7jxxhvVuXNnTZky5Yz7v/76a3Xp0kVVpxe/na93sg94dDwunFFPLze7BJyWMWmA2SXgtP6XevaRqbiwgi7we9X6z/zca+da/sBPPxvGLB7/3/fwww+rtLT0J/e3bNlSq1at+kVFAQBgpgCm/d117979rPvr1auna67hSVIAgF8vK69dky7AW/0AAIBv4/G+AAAYWLzxJ/wBADCyydrpz7Q/AAB+hs4fAAADVvsDAOBnWO0PAAAshc4fAAADizf+hD8AAEZW/1Q/pv0BAPAzdP4AABhYvPEn/AEAMLL6an/CHwAAA4tnP/f8AQDwN3T+AAAYWH21P+EPAICBtaOfaX8AAPwOnT8AAAZWX+1P5w8AgEGAzXubJ9LT09W1a1eFhoaqUaNGGjx4sLZv3+52TM+ePWWz2dy2MWPGePb9eVYWAAC4UDIzM5WSkqL169drxYoVOnHihPr166fS0lK34+69917l5eW5tueee86j6zDtDwCAgVnT/suWLXN7PX/+fDVq1EhZWVnq0aOHazw4OFjR0dE/+zp0/gAAGNhs3tucTqeKi4vdNqfTeV51FBUVSZLCw8PdxhcsWKDIyEi1b99eaWlpOn78uEffH+EPAMAFlJ6eLofD4balp6ef8+uqqqo0fvx4XX311Wrfvr1rfOjQoXrrrbe0atUqpaWl6e9//7uGDx/uUU1M+wMAYODNaf+0tDSlpqa6jdnt9nN+XUpKir799lutXbvWbfy+++5z/blDhw6KiYlRnz59lJubqxYtWpxXTYQ/AAAGnq7SPxu73X5eYf9jY8eO1fvvv681a9aoSZMmZz02KSlJkpSTk0P4AwDwc5m14K+6ulrjxo3T4sWLtXr1ajVv3vycX5OdnS1JiomJOe/rEP4AAPiIlJQUZWRk6L333lNoaKjy8/MlSQ6HQ3Xr1lVubq4yMjJ0/fXXKyIiQt98840mTJigHj16qGPHjud9HcIfAAADs57vN2vWLEmnHuTzY/PmzdPIkSMVGBiojz/+WNOmTVNpaani4uI0ZMgQPfbYYx5dh/AHAMDArE/1q66uPuv+uLg4ZWZm/uLr8FY/AAD8DJ0/AAAGFv9cH8IfAAAjPtUPAABYCp0/AAAGFm/8CX8AAIzMWu1/sTDtDwCAn6HzBwDAwOKNP+EPAICR1Vf7+0z4d4qub3YJOG3eY/3NLgGn3T7+dbNLwGlrXksxuwT8SNfmjgt6fqvfE7f69wcAAAx8pvMHAMBXMO0PAICfCbB29jPtDwCAv6HzBwDAwOqdP+EPAICB1e/5M+0PAICfofMHAMCAaX8AAPyMxWf9mfYHAMDf0PkDAGBg9Y/0JfwBADCw+rQ44Q8AgIHFG3/L/3IDAAAM6PwBADDgnj8AAH7G4tnPtD8AAP6Gzh8AAAOe8AcAgJ+x+j1/pv0BAPAzdP4AABhYvPGn8wcAwCjA5r3NE+np6eratatCQ0PVqFEjDR48WNu3b3c7pry8XCkpKYqIiFBISIiGDBmigoICz74/z8oCAAAXSmZmplJSUrR+/XqtWLFCJ06cUL9+/VRaWuo6ZsKECVq6dKkWLVqkzMxMHThwQDfffLNH12HaHwAAA5vMmfdftmyZ2+v58+erUaNGysrKUo8ePVRUVKS5c+cqIyNDvXv3liTNmzdPl156qdavX68rr7zyvK5D+AMAYODNt/o5nU45nU63MbvdLrvdfs6vLSoqkiSFh4dLkrKysnTixAn17dvXdUxCQoLi4+O1bt268w5/pv0BADDw5j3/9PR0ORwOty09Pf2cNVRVVWn8+PG6+uqr1b59e0lSfn6+AgMDVb9+fbdjo6KilJ+ff97fH50/AAAXUFpamlJTU93GzqfrT0lJ0bfffqu1a9d6vSbCHwAAA5sX3+t3vlP8PzZ27Fi9//77WrNmjZo0aeIaj46OVkVFhQoLC926/4KCAkVHR5/3+Zn2BwDAwKy3+lVXV2vs2LFavHixPvnkEzVv3txtf2JiourUqaOVK1e6xrZv3669e/cqOTn5vK9D5w8AgI9ISUlRRkaG3nvvPYWGhrru4zscDtWtW1cOh0OjR49WamqqwsPDFRYWpnHjxik5Ofm8F/tJhD8AADWY9YS/WbNmSZJ69uzpNj5v3jyNHDlSkvTSSy8pICBAQ4YMkdPpVP/+/TVz5kyPrkP4AwBgYNYH+1RXV5/zmKCgIM2YMUMzZsz42dfhnj8AAH6Gzh8AAANvPuTHFxH+AAAY8Kl+AADAUuj8AQAwCDDpg30uFsIfAAADq0/7E/4AABhYfcEf9/wBAPAzdP6/UGVlpd6e/6pWf/SBCo8eUXhkQ/W+bqBuu+ter34wBGraveVrrV36Dx3YtUPHfjiiOx56Sm27dnM75uD+Pfoo42/aveVrVVVVqlHjprr9D5NVPzLKpKr9w0O3J+upe3tp+j+/0MMzP1aD0CA9PqKH+lzeXHGNwnS48LiW/t8OTZ6/RsWlznOfEB7ZtulL/efdt7Rr5zYVHj2s8U88p8uv6unav2HtKq384F/avXOrSo4V65kZb6lpi9bmFeyDzHrIz8VC+P9C/8yYrw/fe1fj06YovlkL5WzfrFemPql69UI08LdDzS7P0iqc5Ypu2kKX9Rqgt194osb+o/nf67VJv1dirwHqfctIBdUNVsH+3apdJ9CEav1HYpsYjf5NF32TW+Aai4kIVUxEiNJmr9TW3YcVH+XQXydcp5jIUA2d/C8Tq7UmZ3m54pu3Uo9+A/XyUxPPsL9Mbdp1UlL3Ppr78rMmVOj7LJ79hP8vtW3z10q6+hp1Te4uSYqKidWalcu0Y9tmkyuzvtZdktS6S9JP7l+xcK5ad0lS/+FjXGPh0Y0vRml+q15QHc3744164MUP9Oiwq13jW3Yf0h0/CvldeYV6cm6mXk+7UbUCbKqsOvcjTXH+OnW9Sp26XvWT+7v1vV6SdCj/wMUqCT6Ge/6/UEK7Tvrmyy/0/b49kqRdOdu1ZVO2EpOuPsdX4kKqqqrSjq/WKyKmid545mFNvfcmzf7T/dqyYa3ZpVnatAf7a9n6XK36cvc5jw0Lsav4eAXBD58UYLN5bfNFdP6/0G+HjVLZ8RI9cOdNCgiopaqqSg2/J0U9r73e7NL8WmlxoSrKy/Tpe2+r7213q9+w32ln9hda+MITGvXEi2retrPZJVrOLb3aqnPLaHV7YN45j40Iq6u04d30+n++ugiVAZ7z0cz2Go/Dv6ysTFlZWQoPD1fbtm3d9pWXl+udd97RXXfdddZzOJ1OOZ3ui3wqnJUKtNs9Lcd0a1d9pMwVH+oPjz+r+GYttCtnu16b/heFRzZUn+tuNLs8v1VdVSVJSrj8Kl11wy2SpJhmLbV3x2ZtWLGU8PeyJg1D9XzKtfrNIxlynqg867GhwYFa/Oyt2rrnsJ5+49OLVCGAH/No2n/Hjh269NJL1aNHD3Xo0EHXXHON8vLyXPuLioo0atSoc54nPT1dDofDbZv91794Xr0PmD9rmoYMG6Uefa5Tsxat1Kv/b3TjLcP07oJzdz+4cILDHAqoVUuNGjdzG2/YOF5FhwvO/EX42bq0jlFUg3pa9+poHfvoUR376FH16NxUD9zUVcc+elQBp980HVI3UP+eeruOHa/QbU+8q5OVVSZXDpxZgBc3X+RR5z9x4kS1b99eGzduVGFhocaPH6+rr75aq1evVnx8/HmfJy0tTampqW5je344e7fgq5zO8hpv6QsICHB1njBH7dp11LhFgg7n7XMbP5K3X46GvM3P21Z9uVuJo+e4jf3t4Ru0fd8RvbBwvaqqqhUaHKilf75dzopK/fbxReecIQDMZPW3ansU/p999pk+/vhjRUZGKjIyUkuXLtUDDzyg7t27a9WqVapXr955ncdut8tumOIPPH7ck1J8RteremjRW3PVMCpG8c1a6Lud2/TeO2+p7/WDzS7N8pzlZTqa/73rdeHBPOXtzlHdkFDVj4xSt4G36Z1pU9Ts0o5q3q6LdmZ/oe1Zn+nuSdPMK9qiSsoqtGX3Ibex0vITOlpcpi27Dyk0OFDv//kO1Q2qo1HPvquwYLvCgk/9N+BQ0XFVsejPq8rLjqvgwH7X60P5B7Qnd4fqhYYpslG0So4V6cjBAv1w5NTPLG//qQXLjgbhqh8eaUrNuLg8Cv+ysjLVrv2/L7HZbJo1a5bGjh2ra665RhkZGV4v0Nfd9+BELZg7U6++9KyKfvhB4ZENdd2Nv9VtI+4zuzTLO5C7Xa9PmeB6/eGbMyVJXa7pr5sfeFRtr+iugfdO0JolGfrPvL8qMjZOt6dOVtOEDmaV7Lc6t4rWFW1Pvc1yy1sPuO1rM3SG9hYUmVGWZX23Y6uenXi/6/WCv02TJHXve4N+99AkfbnuU/3txSmu/dPT/yRJumnYPRpyJ//tkmTxj/WRbNXV1ef9K/cVV1yhcePG6c4776yxb+zYsVqwYIGKi4tVWen5dN72/F9n529FX+cXml0CThvxh/lml4DT1ryWYnYJ+JGuzR0X9PxvZe0/90HnaXhiE6+dy1s8Wotw00036e233z7jvunTp+uOO+6QB79LAADgk2xe3HyRR+GflpamDz744Cf3z5w5U1UsdAMAwKfxkB8AAAwsvtif8AcAwMjqb/Xz1ecPAACAC4TOHwAAA6t3xoQ/AAAGTPsDAABLofMHAMDA2n0/4Q8AQA1M+wMAAEuh8wcAwMDqnbHVvz8AADxms9m8tnlizZo1GjhwoGJjY2Wz2bRkyRK3/SNHjqxx/uuuu87j74/wBwDAwKwP9iktLVWnTp00Y8aMnzzmuuuuU15enmv7qQ/cOxum/QEA8BEDBgzQgAEDznqM3W5XdHT0L7oOnT8AAAY2m/c2p9Op4uJit83pdP7s2lavXq1GjRqpTZs2uv/++3XkyBGPz0H4AwBgECCb17b09HQ5HA63LT09/WfVdd111+nNN9/UypUr9ec//1mZmZkaMGCAKisrPToP0/4AAFxAaWlpSk1NdRuz2+0/61y33367688dOnRQx44d1aJFC61evVp9+vQ57/MQ/gAAGHjzGT92u/1nh/25XHLJJYqMjFROTg7hDwDAL2H7lTzgd//+/Tpy5IhiYmI8+jrCHwAAH1FSUqKcnBzX6127dik7O1vh4eEKDw/X5MmTNWTIEEVHRys3N1ePPPKIWrZsqf79+3t0HcIfAAADsx7tv3HjRvXq1cv1+r9rBUaMGKFZs2bpm2++0RtvvKHCwkLFxsaqX79+euqppzy+rUD4AwBgEGDStH/Pnj1VXV39k/uXL1/ulevwVj8AAPwMnT8AAAYW/0Rfwh8AACPCHwAAP/Nreavfz8U9fwAA/AydPwAABgHWbvwJfwAAjJj2BwAAlkLnDwCAAav9AQDwM0z7AwAAS6HzBwDAgNX+AAD4Gab9AQCApdD5AwBgwGp/AAD8jMWzn/AHAMAowOKtP/f8AQDwMz7T+YfXCzS7BJzW3FHP7BJw2hsvjDS7BJzW4+Y/mV0CfqTsq+kX9PzW7vt9KPwBAPAZFk9/pv0BAPAzdP4AABhY/SE/hD8AAAYWX+zPtD8AAP6Gzh8AAAOLN/6EPwAANVg8/Zn2BwDAz9D5AwBgwGp/AAD8jNVX+xP+AAAYWDz7uecPAIC/IfwBADCyeXHzwJo1azRw4EDFxsbKZrNpyZIlbvurq6v1xBNPKCYmRnXr1lXfvn21c+dOj789wh8AAAObF//nidLSUnXq1EkzZsw44/7nnntOr7zyil599VV9/vnnqlevnvr376/y8nKPrsM9fwAAfMSAAQM0YMCAM+6rrq7WtGnT9Nhjj2nQoEGSpDfffFNRUVFasmSJbr/99vO+Dp0/AAAGNpv3NqfTqeLiYrfN6XR6XNOuXbuUn5+vvn37usYcDoeSkpK0bt06j85F+AMAYODNW/7p6elyOBxuW3p6usc15efnS5KioqLcxqOiolz7zhfT/gAAXEBpaWlKTU11G7Pb7SZVcwrhDwCAkRff6G+3270S9tHR0ZKkgoICxcTEuMYLCgrUuXNnj87FtD8AAAZmrfY/m+bNmys6OlorV650jRUXF+vzzz9XcnKyR+ei8wcAwEeUlJQoJyfH9XrXrl3Kzs5WeHi44uPjNX78eD399NNq1aqVmjdvrscff1yxsbEaPHiwR9ch/AEAMDDr2f4bN25Ur169XK//u1ZgxIgRmj9/vh555BGVlpbqvvvuU2Fhobp166Zly5YpKCjIo+vYqqurq71a+c906NhJs0vAabsPl5pdAk7bVcTPwleMGPWs2SXgR8q+mn5Bz//t/hKvnat9kxCvnctb6PwBADCy+Cf7sOAPAAA/Q+cPAICBN1fp+yLCHwAAA7MW/F0sTPsDAOBn6PwBADCweONP+AMAUIPF059pfwAA/AydPwAABqz2BwDAz7DaHwAAWAqdPwAABhZv/Al/AABqsHj6E/4AABhYfcEf9/wBAPAzdP4AABhYfbU/4Q8AgIHFs5/w97a/z5+j2dOn6ZY7huvBP6SZXY6lbdv0pf7z7lvatXObCo8e1vgnntPlV/V07d+wdpVWfvAv7d65VSXHivXMjLfUtEVr8wq2sN1bvtbapf/QgV07dOyHI7rjoafUtms3t2MO7t+jjzL+pt1bvlZVVaUaNW6q2/8wWfUjo0yq2j88NOpaPfX7QZq+YJUe/ss/JUnL5zyoHpe3cjtuzrtr9ftnFppRIkxA+HvR1s2b9O9/LVKLVgTMxeAsL1d881bq0W+gXn5q4hn2l6lNu05K6t5Hc19+1oQK/UeFs1zRTVvosl4D9PYLT9TYfzT/e7026fdK7DVAvW8ZqaC6wSrYv1u16wSaUK3/SGwbr9FDrtY3O/bX2Df3n/+np2a973p9vPzExSzN91m89Sf8veT48VJNfnyiHvnTZL0xd7bZ5fiFTl2vUqeuV/3k/m59r5ckHco/cLFK8lutuySpdZekn9y/YuFcte6SpP7Dx7jGwqMbX4zS/Fa9uoGa9+xIPfDU23r0nutq7C8rr1DBkWMmVPbrwGp/nJcX//y0rrq6h7omJZtdCuBTqqqqtOOr9YqIaaI3nnlYU++9SbP/dL+2bFhrdmmWNi3tNi379Fut+nz7Gfffdv3l2vfJVG1c9EdNGXej6gbVucgVwkwed/5bt27V+vXrlZycrISEBG3btk0vv/yynE6nhg8frt69e5/zHE6nU06n032sopbsdrun5fiEj5d/oB3btmrOm/8wuxTA55QWF6qivEyfvve2+t52t/oN+512Zn+hhS88oVFPvKjmbTubXaLl3NI/UZ0T4tRt+HNn3P+PDzdqb95R5R0qUodWsXr6wUFq3bSRbn/otYtcqe9itf+PLFu2TIMGDVJISIiOHz+uxYsX66677lKnTp1UVVWlfv366aOPPjrnLwDp6emaPHmy29hDjz6uR/5Y816hryvIz9PLL0zVSzPm/Gp/eQEupOqqKklSwuVX6aobbpEkxTRrqb07NmvDiqWEv5c1iaqv5x8eot/cP13OipNnPOb1f/2f68+bcw4o73Cxlv3t92reJFK79h++WKX6NItnv2fhP2XKFD388MN6+umntXDhQg0dOlT333+/nnnmGUlSWlqapk6des7wT0tLU2pqqttYcUUtD0v3Ddu3bdEPR49o9PBbXGOVlZX6+quN+tc7b+uTz75SrVq/zu8N8IbgMIcCatVSo8bN3MYbNo7X3m2bzCnKwrpcGq+oiDCty/jfItjatWup22UtNOa2HnIkjVdVVbXb12zYtFuS1CKuIeHvJzwK/82bN+vNN9+UJN16662688479dvf/ta1f9iwYZo3b945z2O322t0yc5jZ/4N1ddd3vVKvblwidvYs1P+pKZNL9GwEaMJfvi92rXrqHGLBB3O2+c2fiRvvxwNeZuft636YrsSf/uM29jfJg/X9l0FemH+ihrBL0md2jSRJOUfLrooNf4qWLz19/iev+30jZCAgAAFBQXJ4XC49oWGhqqoyL/+8gTXq6dLWrq/XzYoKFhh9R01xuFd5WXHVXDgf29hOpR/QHtyd6heaJgiG0Wr5FiRjhws0A9HDkmS8vbvkSQ5GoSrfnikKTVblbO8TEfzv3e9LjyYp7zdOaobEqr6kVHqNvA2vTNtippd2lHN23XRzuwvtD3rM909aZp5RVtUyXGntuTmuY2VllXoaFGptuTmqXmTSN024HItX7tZRwpL1aF1Yz33h5v1adZOfbuTd8b8l9VX+3sU/s2aNdPOnTvVokULSdK6desUHx/v2r93717FxMR4t0LgJ3y3Y6uenXi/6/WCv02TJHXve4N+99AkfbnuU/3txSmu/dPT/yRJumnYPRpy530XtVarO5C7Xa9PmeB6/eGbMyVJXa7pr5sfeFRtr+iugfdO0JolGfrPvL8qMjZOt6dOVtOEDmaV7LdOnDip3kltNHZoL9WrG6j9BT9oycpsTX1tudml+RSrL/izVVdX15wD+gmvvvqq4uLidMMNN5xx/x//+EcdPHhQr73m+YrRQ7/SaX8r2n241OwScNquIn4WvmLEKB4U5UvKvpp+Qc+/96jz3Aedp/hw31sM7lHnP2bMmLPuf/ZZ/nEAAH79LN7484Q/AACMrD7tzxP+AADwM4Q/AAA12Ly4nb8nn3xSNpvNbUtISPDKd/RjTPsDAGBg5rR/u3bt9PHHH7te167t/agm/AEA8CG1a9dWdHT0Bb0G0/4AABh4c9Lf6XSquLjYbTN+uN2P7dy5U7Gxsbrkkks0bNgw7d271+vfH+EPAICBzea9LT09XQ6Hw21LT08/43WTkpI0f/58LVu2TLNmzdKuXbvUvXt3HTt2zLvfnycP+bmQeMiP7+AhP76Dh/z4Dh7y41su9EN+8ooqvHau8KDqGp3+mT7j5kwKCwvVtGlTvfjiixo9erTXauKePwAABt58tr/dHvizP/K9fv36at26tXJycrxWj8S0PwAANZnzTr8aSkpKlJub6/XPzSH8AQAwMCv7H3roIWVmZmr37t367LPPdNNNN6lWrVq64447vPBd/Q/T/gAA+Ij9+/frjjvu0JEjR9SwYUN169ZN69evV8OGDb16HcIfAAADsx7ys3DhwotyHcIfAAADby7480Xc8wcAwM/Q+QMAYGTtxp/wBwDAyOLZz7Q/AAD+hs4fAAADMz/S92Ig/AEAMGC1PwAAsBQ6fwAADKw+7U/nDwCAn6HzBwDAgM4fAABYCp0/AAAGVl/tT/gDAGDAtD8AALAUOn8AAAws3vgT/gAA1GDx9GfaHwAAP0PnDwCAAav9AQDwM6z2BwAAlkLnDwCAgcUbf8IfAIAaLJ7+hD8AAAZWX/DHPX8AAPwMnT8AAAZWX+1vq66urja7CCtwOp1KT09XWlqa7Ha72eX4PX4evoOfhe/gZ4H/Ivy9pLi4WA6HQ0VFRQoLCzO7HL/Hz8N38LPwHfws8F/c8wcAwM8Q/gAA+BnCHwAAP0P4e4ndbtekSZNYROMj+Hn4Dn4WvoOfBf6LBX8AAPgZOn8AAPwM4Q8AgJ8h/AEA8DOEPwAAfobwBwDAzxD+XjJjxgw1a9ZMQUFBSkpK0hdffGF2SX5pzZo1GjhwoGJjY2Wz2bRkyRKzS/JL6enp6tq1q0JDQ9WoUSMNHjxY27dvN7ssvzVr1ix17NhRYWFhCgsLU3Jysj788EOzy4KJCH8v+Mc//qHU1FRNmjRJX375pTp16qT+/fvr4MGDZpfmd0pLS9WpUyfNmDHD7FL8WmZmplJSUrR+/XqtWLFCJ06cUL9+/VRaWmp2aX6pSZMmmjp1qrKysrRx40b17t1bgwYN0ubNm80uDSbhff5ekJSUpK5du2r69OmSpKqqKsXFxWncuHF69NFHTa7Of9lsNi1evFiDBw82uxS/d+jQITVq1EiZmZnq0aOH2eVAUnh4uJ5//nmNHj3a7FJgAjr/X6iiokJZWVnq27evaywgIEB9+/bVunXrTKwM8B1FRUWSTgUOzFVZWamFCxeqtLRUycnJZpcDk9Q2u4Bfu8OHD6uyslJRUVFu41FRUdq2bZtJVQG+o6qqSuPHj9fVV1+t9u3bm12O39q0aZOSk5NVXl6ukJAQLV68WG3btjW7LJiE8AdwQaWkpOjbb7/V2rVrzS7Fr7Vp00bZ2dkqKirSu+++qxEjRigzM5NfAPwU4f8LRUZGqlatWiooKHAbLygoUHR0tElVAb5h7Nixev/997VmzRo1adLE7HL8WmBgoFq2bClJSkxM1IYNG/Tyyy9r9uzZJlcGM3DP/xcKDAxUYmKiVq5c6RqrqqrSypUruZ8Gv1VdXa2xY8dq8eLF+uSTT9S8eXOzS4JBVVWVnE6n2WXAJHT+XpCamqoRI0bo8ssv1xVXXKFp06aptLRUo0aNMrs0v1NSUqKcnBzX6127dik7O1vh4eGKj483sTL/kpKSooyMDL333nsKDQ1Vfn6+JMnhcKhu3bomV+d/0tLSNGDAAMXHx+vYsWPKyMjQ6tWrtXz5crNLg0l4q5+XTJ8+Xc8//7zy8/PVuXNnvfLKK0pKSjK7LL+zevVq9erVq8b4iBEjNH/+/ItfkJ+y2WxnHJ83b55Gjhx5cYuBRo8erZUrVyovL08Oh0MdO3bUxIkTde2115pdGkxC+AMA4Ge45w8AgJ8h/AEA8DOEPwAAfobwBwDAzxD+AAD4GcIfAAA/Q/gDAOBnCH8AAPwM4Q8AgJ8h/AEA8DOEPwAAfub/AcqIF/KpIXppAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# confusion matrix of t-SNE\n",
    "values_tsne = values_train.copy()\n",
    "values_tsne.drop(columns=[\"path\", \"true\", \"predicted\", \"correct\"], inplace=True)\n",
    "y_true = []\n",
    "y_pred = []\n",
    "for i in tqdm(range(len(values_test))):\n",
    "    element = values_test.iloc[i]\n",
    "    sm0 = element[\"sm0\"]\n",
    "    sm1 = element[\"sm1\"]\n",
    "    sm2 = element[\"sm2\"]\n",
    "    sm3 = element[\"sm3\"]\n",
    "    values_tsne = pd.concat([values_tsne, pd.DataFrame([[sm0, sm1, sm2, sm3]], columns=[\"sm0\", \"sm1\", \"sm2\", \"sm3\"])])\n",
    "    points = values_tsne[[\"sm0\", \"sm1\", \"sm2\", \"sm3\"]].values\n",
    "    tsne = TSNE(n_components=2, verbose=0, perplexity=40, n_iter=400)\n",
    "    tsne_results = tsne.fit_transform(points)\n",
    "    distances = np.linalg.norm(tsne_results - tsne_results[len(tsne_results)-1], axis=1)\n",
    "    points_num = 12\n",
    "    nearest_points = np.argsort(distances)[:points_num]\n",
    "    nearest_points = np.delete(nearest_points, 0)\n",
    "    scores = []\n",
    "    for k in range(len(nearest_points)):\n",
    "        scores.append(values_train.iloc[nearest_points[k]][\"true\"])\n",
    "    mode_val = np.median(scores)\n",
    "    values_tsne = values_tsne[:-1]\n",
    "    y_true.append(element[\"true\"])\n",
    "    y_pred.append(mode_val)\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.48\n",
      "precision: 0.47247812831389185\n"
     ]
    }
   ],
   "source": [
    "print(\"accuracy:\", accuracy_score(y_true, y_pred))\t\n",
    "print(\"precision:\", precision_score(y_true, y_pred, average=\"macro\"))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FINAL MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MIDfinal2(nn.Module):\n",
    "    def __init__(self, values_train: pd.DataFrame):\n",
    "        super(MIDfinal2, self).__init__()\n",
    "        self.MIDmodel = MIDmodel2()\n",
    "        self.MIDmodel.load_state_dict(torch.load(\"models/MIDmodel2.pt\"))\n",
    "        self.MIDmodel.to(device)\n",
    "        self.MIDmodel.eval()\n",
    "        self.binary_classifier = pickle.load(open(\"models/svc.pkl\", \"rb\"))\n",
    "        self.values_train = values_train\n",
    "\n",
    "    def forward(self, x):\n",
    "        # get the output from the MIDmodel\n",
    "        res = self.MIDmodel(x)\n",
    "        predicted_label =torch.nn.functional.softmax(res, dim=1)\n",
    "        predicted_label = predicted_label.cpu().detach().numpy()\n",
    "        predicted_label = pd.DataFrame(predicted_label, columns=[\"sm0\", \"sm1\", \"sm2\", \"sm3\"])\n",
    "        label_value = torch.argmax(res)\n",
    "\n",
    "        results = []\n",
    "        results.append(label_value)\n",
    "\n",
    "        binary_output = self.binary_classifier.predict(predicted_label)\n",
    "        binary_output = binary_output[0]\n",
    "        results.append(binary_output)\n",
    "\n",
    "        if binary_output == 0:\n",
    "            values_tsne = self.values_train.copy()\n",
    "            values_tsne.drop(columns=[\"path\", \"true\", \"predicted\", \"correct\"], inplace=True)\n",
    "            res = predicted_label\n",
    "            res = pd.DataFrame(res, columns=[\"sm0\", \"sm1\", \"sm2\", \"sm3\"])\n",
    "            values_tsne = pd.concat([values_tsne, res])\n",
    "            points = values_tsne[[\"sm0\", \"sm1\", \"sm2\", \"sm3\"]].values\n",
    "            tsne = TSNE(n_components=2, verbose=0, perplexity=40, n_iter=400)\n",
    "            tsne_results = tsne.fit_transform(points)\n",
    "            distances = np.linalg.norm(tsne_results - tsne_results[len(tsne_results)-1], axis=1)\n",
    "            points_num = 12\n",
    "            nearest_points = np.argsort(distances)[:points_num]\n",
    "            nearest_points = np.delete(nearest_points, 0)\n",
    "            scores = []\n",
    "            for k in range(len(nearest_points)):\n",
    "                scores.append(self.values_train.iloc[nearest_points[k]][\"true\"])\n",
    "            mode_val = max(set(scores), key=scores.count)\n",
    "            results.append(mode_val)\n",
    "            return results\n",
    "        # trasform res to the final label\n",
    "        res = torch.argmax(res).cpu().detach().numpy()\n",
    "        results.append(res)\n",
    "        return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 348/348 [1:15:31<00:00, 13.02s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf8AAAGdCAYAAAAczXrvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyiElEQVR4nO3de1xUdf7H8fcgMiDCKKjgDS9ZankrNEVTk1Czq2XlVrZWbq2bWkqZsV3sYmG1ZZao5c+0m6vZrpVt6ZqVWkkqaXlFLQsVQQFFrqPC/P6onZqDplPIGc95PXvM4+F8z5mZD5KPN5/P+c7g8Hg8HgEAANsIMrsAAABQswh/AABshvAHAMBmCH8AAGyG8AcAwGYIfwAAbIbwBwDAZgh/AABshvAHAMBmgs0u4H/GvrfN7BLws14tIs0uAT/rHhdtdgn42Qfb9pldAn7lrp4tT+vzh50/utqeq2z9tGp7ruoSMOEPAEDAcFh7MG7trw4AAFRB5w8AgJHDYXYFpxXhDwCAkcXH/oQ/AABGFu/8rf2jDQAAqILOHwAAI8b+AADYDGN/AABgJXT+AAAYMfYHAMBmGPsDAAArofMHAMCIsT8AADbD2B8AAFgJnT8AAEaM/QEAsBmLj/0JfwAAjCze+Vv7qwMAAFXQ+QMAYGTxzp/wBwDAKMja1/yt/aMNAACogs4fAAAjxv4AANiMxd/qZ+0fbQAAQBV0/gAAGDH2BwDAZhj7AwAAK6HzBwDAiLE/AAA2Y/GxP+EPAICRxTt/a391AACgCjp/AACMGPsDAGAzjP0BAICV0PkDAGDE2B8AAJth7A8AAKyEzh8AACOLd/6Ev596taynXi3rKapObUlSTtERLc3M09b9JZKk4CCHru7QSBc0jVRwkEPb9pdo4bc5KnZXmFm2Ja1Y9Ja2rFmlA9lZqh3iVNw552nAzXeqYZM4SVJp8WF98vZc7fx2nQ7l5So8sp7ad+ulpKG3K7ROXZOrt5Zv16/TwrfmanvmVhXkHdCjk19Qr76J3uMej0evzZquj97/l4qLinRepy66+/6H1Kx5CxOrtqa1H8zXzowvdDBnt4Jrh6hxm3N10fUjVL9xc+8570wer72Z3/o8rsPFl+mS4ffUdLmBi2v++LVDZce0eMsBHSg5IoekbnEujejeTP/4bJdyio7omg6NdG5MXc1du1dlRyt0XadY3d6tqV78PMvs0i3nh63fqPvAwWp6VltVVlRo2fz/09wn79c9z81RSGiYigrydfhgni69ZaQaNm2hQ3m5ev//pqjoYL5uTH7M7PItpby8TK3PbquBV1yjx1LGVTm+4M05enfhPN3/8CTFNmmqua9MU8rYkZo9712FOJ0mVGxdezO/VedLrlRMq3NUWVGhL/81V4ue+7tueXKWajtDved16DtIPa75s/d+cAjfBzsh/P20ObfY5/6HW/PUq2V9tagfpkNlx9S9RT29sS5bO/JKJUnz1u/T3y9prRb1Q/XjwXIzSras4X9/xuf+kLseUOod12jv99vV6tzOiolrpZvufdx7PDq2qfoPHaGF055SRUWFatWqVdMlW9aFCb11YULv4x7zeDxatOBN3XzrHerZp58kacIjT+r6y/vpi5WfqF//QTVZquUNvvcpn/v9R9yrWfcM1f4fdqhp247e9eAQp8JdUTVd3pmDsT9OxCGpS9MIOWs59MPBMjWvF6rgIIe2HyjxnrO/+IgKSo+qZf0wwv80Ky/96e+9Tt3I3zzHGVaH4K9BOdl7VZCfp/O79fCuhdeNULtzO2rLpm8I/9PsSNlP/y6c4RE+65mrP9W21Z8o3FVfrTr30IVX3eQzGbA9xv6+8vLy9Oqrr2r16tXKycmRJMXGxqpnz5669dZb1bBhw2ovMtA0jnBqbJ8WCg5y6EhFpWav2avcoiNqGhmqYxWVKjtW6XN+kfuYIkP5Oet0qqys1IevTVNc2w6KiWt13HNKDhfq03+/oW5JV9RwdfZWkJ8nSaofFe2zXj8qWgfz880oyTY8lZVa8c+Zanz2eWrQrKV3vW2PfoqMbqTwetHK27NLXyycrYM5e3TFmEfMKzbQmNT5P/roo3rsMd/Lkm3bttW2bdskSeXl5br33ns1f/58ud1uDRw4UNOnT1dMTIxfr+NXIq1du1YDBw5UnTp1lJSUpHPOOUeSlJubqxdffFGTJ0/W0qVL1bVr1998HrfbLbfb7bN27OgRBdcO8at4s+wvduvZz3YpNDhIXZpE6uYLGuulL7imb6YPXp2q3N27dMdjLx33eHlpid54+gE1atZCidfdWrPFASb59M1pyt/zo67/+3M+6x0vvsz75wbNWyncFaV/PztBh/Znq16jJjVdJgzOO+88ffzxx977wcG/RPW4ceP0n//8RwsXLpTL5dLo0aN17bXX6osvvvDrNfwK/zFjxuj666/XzJkz5TCMRDwej0aOHKkxY8Zo9erVv/k8qampVX6y6T50lHrcONqfckxT4ZHySo5KkvYUHlDz+qHq27q+1u8tUnCtIIUFB/l0/xHOYB0uP2ZWuZa3+NWp2vb1av3l0alyRVedPLnLSvVa6gSFhNbRTfc+oVrBTGFqUlR0A0nSwYJ8RTf45ftzsCBfZ53T1qyyLO/TN6Zp14avdF3Kc4qI+u2JbOxZ7SRJhbmEv5eJY//g4GDFxsZWWS8sLNTs2bM1b948JSb+9G6aOXPmqH379kpPT1ePHj2qPOZE/JprfPPNNxo3blyV4Jckh8OhcePGacOGDSd9npSUFBUWFvrcul53pz+lBBSHpOCgIO0+VK5jlR6d3TDce6xR3RBF1amtHw6WmVegRXk8Hi1+daq2rPlctz/8vKIaNa5yTnlpieY+OV61goM17P4nVTvkzJguWUlsk6aKim6g9eu+8q6VlBRr25aNOrdDZxMrsyaPx6NP35im777+Utfe/4xcDauGiNGBrO8kSXXqsQHwfxwOR7Xd3G63Dh8+7HMzTr9/bceOHWrSpIlat26tm2++WVlZP02WMzIydPToUSUlJXnPbdeuneLi4k7adBv5Ff6xsbFas2bNCY+vWbPmlK47OJ1ORUZG+tzOlJH/Fe0bqnV0mKLCaqtxhFNXtG+oNg3qaN2eQpUfq9RXPx7S4A6N1KZBHTVzOXXj+bHaVVDKZr/TYPHsF/TNqmW64e4H5Qyro6JDBSo6VKCjR376R/W/4D/iLtc1fx0vd1mp95zKSj53oTqVlZZq5/Zt2rn9p+uSOdl7tXP7Nu3P2SeHw6Frhg7TvLmv6MtVn2rXzu165vEHFd2goXr1STzJM8Nfn74xTdtWf6JL//qAQsLCVFJYoJLCAh37+d/Fof3Z+ur9t5T7ww4dzsvR9+tX67+znlXTth3VsHlrk6u3ptTUVLlcLp9bamrqcc/t3r275s6dqyVLlmjGjBnatWuXevfuraKiIuXk5CgkJET16tXzeUxMTIx3D96p8mv+ed999+nOO+9URkaGLrnkEm/Q5+bmavny5Zo1a5b+8Y9/+FXAmaaus5aGXdBEkc5aKjtWqezDbs1cvVvbD/z01r5Fm/arUtJt3Zp6P+TnnW/9+6bg1KxZ9r4kafZjvu8rv/ZvE3TBxZcqe9cO7dm5VZI05Z5hPufc+9I/Vb/RyTsinJrt2zbrvlEjvPdnvvisJKn/ZVfp/ocnaeiw21ReVqYXJj+u4uIideh0vlKnzOA9/qfBxk8/kCT96+nxPuv9R9yrcy8aoFq1grV7y3pt+O8iHXWXq25UQ7XpepG6XXmjGeUGrONNuH+vlJQUJScn+6w5T/D//qBBv7z7pVOnTurevbtatGiht99+W2FhYdVWk8Pj8Xj8ecCCBQs0ZcoUZWRkqKLip+6pVq1aio+PV3Jysm644YbfVcjY97b9rseh+vVqceK3yqFmdY+LPvlJqBEfbNtndgn4lbt6tjytzx9+/Zxqe66Shbf9ocd369ZNSUlJ6t+/vy655BIdPHjQp/tv0aKFxo4dq3Hjqn7A1on4/V6GoUOHKj09XaWlpdq7d6/27t2r0tJSpaen/+7gBwAAVRUXF+u7775T48aNFR8fr9q1a2v58uXe45mZmcrKylJCQoJfz/u7tz3Xrl1bjRtX3WAFAMCZrjrH/v647777dOWVV6pFixbKzs7WxIkTVatWLd14441yuVwaMWKEkpOTFRUVpcjISI0ZM0YJCQl+7fSX+IQ/AACqMCv89+zZoxtvvFH5+flq2LChLrroIqWnp3s/QG/KlCkKCgrSkCFDfD7kx1+EPwAAAWL+/Pm/eTw0NFRpaWlKS0v7Q69D+AMAYGBW519TCH8AAAwIfwAA7Mba2e//W/0AAMCZjc4fAAADxv4AANiM1cOfsT8AADZD5w8AgIHVO3/CHwAAA6uHP2N/AABshs4fAAAjazf+hD8AAEaM/QEAgKXQ+QMAYGD1zp/wBwDAgPAHAMBurJ39XPMHAMBu6PwBADBg7A8AgM1YPfwZ+wMAYDN0/gAAGFi98yf8AQAwsHr4M/YHAMBm6PwBADCyduNP+AMAYMTYHwAAWAqdPwAABlbv/Al/AAAMCH8AAOzG2tnPNX8AAOyGzh8AAAPG/gAA2IzVw5+xPwAANkPnDwCAgdU7f8IfAAADq4c/Y38AAGyGzh8AACNrN/6BE/5/6hBrdgn4Wb9bnjG7BPxs83uPmF0Cfnb7hS3NLgE1iLE/AACwlIDp/AEACBRW7/wJfwAADCye/YQ/AABGVu/8ueYPAIDN0PkDAGBg8caf8AcAwIixPwAAsBQ6fwAADCze+BP+AAAYBQVZO/0Z+wMAYDN0/gAAGDD2BwDAZtjtDwAALIXOHwAAA4s3/oQ/AABGVh/7E/4AABhYPfy55g8AgM3Q+QMAYGDxxp/wBwDAiLE/AACwFMIfAAADh6P6br/X5MmT5XA4NHbsWO9aeXm5Ro0apejoaNWtW1dDhgxRbm6u389N+AMAYOBwOKrt9nusXbtWL7/8sjp16uSzPm7cOC1evFgLFy7UihUrlJ2drWuvvdbv5yf8AQAIIMXFxbr55ps1a9Ys1a9f37teWFio2bNn6/nnn1diYqLi4+M1Z84cffnll0pPT/frNQh/AAAMqnPs73a7dfjwYZ+b2+0+4WuPGjVKl19+uZKSknzWMzIydPToUZ/1du3aKS4uTqtXr/br6yP8AQAwqM6xf2pqqlwul88tNTX1uK87f/58ff3118c9npOTo5CQENWrV89nPSYmRjk5OX59fbzVDwCA0yglJUXJyck+a06ns8p5u3fv1j333KNly5YpNDT0tNZE+AMAYFCdb/N3Op3HDXujjIwM7d+/XxdccIF3raKiQitXrtS0adO0dOlSHTlyRIcOHfLp/nNzcxUbG+tXTYQ/AAAGZnzIzyWXXKKNGzf6rN12221q166dJkyYoObNm6t27dpavny5hgwZIknKzMxUVlaWEhIS/Hotwh8AAAMzPuAvIiJCHTp08FkLDw9XdHS0d33EiBFKTk5WVFSUIiMjNWbMGCUkJKhHjx5+vRbhDwDAGWLKlCkKCgrSkCFD5Ha7NXDgQE2fPt3v5yH8AQAwCJTP9v/ss8987oeGhiotLU1paWl/6HkJfwAADAIk+08b3ucPAIDN0PkDAGAQKGP/04XwBwDAwOLZz9gfAAC7ofMHAMCAsT8AADZj9fBn7A8AgM3Q+ftp28b1+uhfb+qHndt0qCBPdz/0jOJ79vUeH35Z9+M+bujto3XZdbfUVJm28OCIS/TQiEt81jJ/PKAuN06RJC2d9hf1uaC1z/FZi77S3c++V2M12sXGDRl6Z95c7czcqoL8A3r4qSnq2SfRe/yLFR/rP+8u1M7MrSo6XKhpcxborLPbmVixfc2e9YpefOE53Tzsz7o/5UGzywlYFm/8CX9/ucvL1LzV2eo94Eq9NGlCleNT3/zQ5/63677Uq1OfVNdeiVXOxR+3+ftcXX73bO/9YxWVPsdnv7dGT8z62Hu/tPxojdVmJ+VlZWrdpq0GXD5Ykx5MPu7x8zqdrz6JAzX16cdMqBCStGnjt3pn4Xydc05bs0sJeFYf+xP+furcrac6d+t5wuP1oqJ97q9PX6n2neLVqHHT012aLR07VqHcguITHi8rP/qbx1E9uiVcpG4JF53w+CWXXilJyt23t6ZKgkFpSYlSJozXxMcmadbLM8wuJ+BZPPu55n86FR7M1zdrv1CfAVeZXYpltWneQN+/94C2LLxPcybeoOYxLp/jQwd00e4PH9S6N+/R4yMHKMxZ26RKAXM9Nelx9enTVz0STty8wD6qvfPfvXu3Jk6cqFdfffWE57jdbrndbp+1I263QpzO6i7HVJ9//KFCw8IV3+tis0uxpLWbd+vOSe9oe1aeYhtE6MHbE/XxjDsVP2yqikuPaMGyb5SVc0j7DhxWxzaxmnTXpTonrqH+9Pe3zC4dqFEfffgfbd26RfMWvGN2KWcMq4/9q73zLygo0Guvvfab56SmpsrlcvncXp85pbpLMd2qZYuV0G+gQkKs9UNNoPhv+nb9+9NN2vRdjj7+aocG3/uaXHXDNCSxoyTp1ffW6uOvdmjz97ma/99vNOKJhbr64vPUqmmUyZUDNSdn3z49M/lJpT79rJwWa7BOJ4ej+m6ByO/O//333//N499///1JnyMlJUXJyb6bgjbsKfO3lICWuWm99u35UXc9MMnsUmyjsLhcO3fn6axm0cc9vnbzbknSWc2itWtvQU2WBphmy5bNKsjP15+uv9a7VlFRoYx1azX/n29p7fqNqlWrlokVwgx+h//gwYPlcDjk8XhOeM7JxiVOp7PKT6AhzsoTnH1mWvnfxWrZpp3iWp9jdim2ER4WolZNo5SzpOi4xzuf3ViSlJN3/OOAFXXv0UPvvLvYZ23igylq2bq1bhtxB8F/AkGB2rJXE7/Dv3Hjxpo+fbquvvrq4x7fsGGD4uPj/3Bhgaq8rFS52Xu89w/kZuvH77arbkSkohvFSpLKSou1ZtVy3fiXe8wq0xZSRw/Sfz7fpqycg2rSIFIP/eUSVVR49Payb9WqaZSG9u+spaszlV9Yqo5tYvXMPZdr1fpd2vRdjtmlW05Zaamy92Z57+fu26vvdmxTRIRLjWIbq+hwofbn7lN+3gFJ0p6sHyRJ9aMaKCq6gRkl20Z4eF2dfbZvExJWp47quepVWccvLJ79/od/fHy8MjIyThj+J5sKnOl27diqyQ/c5b3/z1kvSJIuSrpcdyQ/IklKX7FMkkc9Lh5gQoX20bSRS68/NlRRrjrKO1SiL7/9UX3vnKG8QyUKDQlWYrc2Gj20l8JDa2vP/kK9++lmTZ77qdllW9KObZs14e6/eO+/8tI/JElJg67SvQ8+ofTPP9PzTz3iPT554k+fkXHzbSM1bMTfarZYAHJ4/EzqVatWqaSkRJdeeulxj5eUlGjdunXq27fvcY+fSPp3h/w6H6dPv1ueMbsE/Gzze4+c/CTUiCb1Q80uAb8Sepo/pWbg9K+q7bmW3nX8T341k99/fb179/7N4+Hh4X4HPwAAgSSIsT8AAPbC+/wBAICl0PkDAGBg8caf8AcAwMgha6c/Y38AAGyGzh8AAAN2+wMAYDPs9gcAAJZC5w8AgIHFG3/CHwAAI6v/Vj/G/gAA2AydPwAABhZv/Al/AACMrL7bn/AHAMDA4tnPNX8AAOyGzh8AAAOr7/Yn/AEAMLB29DP2BwDAduj8AQAwYLc/AAA2Y/Xf6sfYHwAAm6HzBwDAgLE/AAA2Y/HsZ+wPAIDd0PkDAGDA2B8AAJux+m5/wh8AAAOrd/5c8wcAwGbo/AEAMLB230/4AwBQhdV/qx9jfwAAbIbOHwAAA4s3/oQ/AABG7PYHAACWQucPAICBxRt/wh8AACN2+wMAAEuh8wcAwMDijT/hDwCAkdV3+wdM+DePrmN2CfjZp2/cb3YJ+FmXMW+bXQJ+tm7qdWaXgF85J+b0ZoZZ18RnzJihGTNm6IcffpAknXfeeXrkkUc0aNAgSVJ5ebnuvfdezZ8/X263WwMHDtT06dMVExPj1+twzR8AgADRrFkzTZ48WRkZGVq3bp0SExN19dVXa/PmzZKkcePGafHixVq4cKFWrFih7OxsXXvttX6/TsB0/gAABAqzxv5XXnmlz/0nn3xSM2bMUHp6upo1a6bZs2dr3rx5SkxMlCTNmTNH7du3V3p6unr06HHKr0P4AwBgEFSN2e92u+V2u33WnE6nnE7nbz6uoqJCCxcuVElJiRISEpSRkaGjR48qKSnJe067du0UFxen1atX+xX+jP0BADiNUlNT5XK5fG6pqaknPH/jxo2qW7eunE6nRo4cqUWLFuncc89VTk6OQkJCVK9ePZ/zY2JilJOT41dNdP4AABhUZ+efkpKi5ORkn7Xf6vrbtm2rDRs2qLCwUO+8846GDx+uFStWVF9BIvwBAKiiOq/5n8qI/9dCQkLUpk0bSVJ8fLzWrl2rqVOnaujQoTpy5IgOHTrk0/3n5uYqNjbWr5oY+wMAEMAqKyvldrsVHx+v2rVra/ny5d5jmZmZysrKUkJCgl/PSecPAIBBdY79/ZGSkqJBgwYpLi5ORUVFmjdvnj777DMtXbpULpdLI0aMUHJysqKiohQZGakxY8YoISHBr81+EuEPAEAVZn3A3/79+/XnP/9Z+/btk8vlUqdOnbR06VL1799fkjRlyhQFBQVpyJAhPh/y4y/CHwCAADF79uzfPB4aGqq0tDSlpaX9odch/AEAMLD6r/Ql/AEAMLD6bnjCHwAAA4s3/pb/4QYAABjQ+QMAYMA1fwAAbMbi2c/YHwAAu6HzBwDAwKxP+KsphD8AAAZWv+bP2B8AAJuh8wcAwMDijT/hDwCAkdWv+TP2BwDAZuj8AQAwcMjarT/hDwCAgdXH/oQ/AAAGVg9/rvkDAGAzdP4AABg4LP5eP8IfAAADxv4AAMBS6PwBADCw+NSf8AcAwIhf7AMAACyFzh8AAAOrb/gj/AEAMLD41J+xPwAAdkPnDwCAQRC/2AcAAHux+tif8AcAwMDqG/645g8AgM3Q+VeD0pISvfryNH2+YrkOHSxQm3PaaXTyA2p3bgezS7O0bRvX66N/vakfdm7ToYI83f3QM4rv2dd7fPhl3Y/7uKG3j9Zl191SU2Xa0rirOuixmy7Q9A+36IHX10mSWsXU1aSbuyqhXSOFBAfp42+yNX7uGh0oLDe5WuvZtCFD/57/ur7L3KKC/Dz9/cnnldC7nyTp2LGjenPWdK1L/1w5+/YoPLyuOnftruF/vVvRDRqZXHng4EN+cFL/eGqiMtasVsqjT2n2W/9W1+49NX70HTqwP9fs0izNXV6m5q3O1i13jT/u8alvfuhzGzH2ITkcDnXtlVjDldrLBa2jdVvS2dr4Y4F3rY4zWO/+vb888uiKJ/6rAROXKCQ4SG+PT7T8tVUzlJeXqdVZ52jkuJQqx9zl5fpux1YNHX6HXvi/fypl0nPam/WjJqWMrflCA5jDUX23QETn/we5y8u18tOPNemZF9X5/K6SpFvvuEurV32m9/+9QCNG3m1ugRbWuVtPde7W84TH60VF+9xfn75S7TvFq1Hjpqe7NNsKdwbr/8b01t2vpGv8tR296z3aNlRcw3Bd9MAHKio7KkkaOf0LZc3+k/qe11ifbdpnVsmW1LXHRera46LjHguvG6Ennp/ps/bXsQ/o3r8O0/7cfWoU07gmSoTJ6Pz/oIqKClVWVCjEGeKz7nSGatM3602qCkaFB/P1zdov1GfAVWaXYmnP3d5dS9fvqRLmIcG15PFI7qMV3rXyoxWq9HiU0I5Rs9lKS4rkcDhUt26E2aUEjCCHo9pugcjv8C8rK9Pnn3+uLVu2VDlWXl6u119/vVoKO1PUCQ/XuR07641XX1begf2qqKjQso8Wa8umb5Sfl2d2efjZ5x9/qNCwcMX3utjsUixrSEJLdW4VpUf/+XWVY2t3HFCJ+5gev+kChYXUUh1nsJ4c1lXBtYIUUy/MhGrxP0fcbs2d+aL6XHKp6oTXNbucgGH1sb9f4b99+3a1b99effr0UceOHdW3b1/t2/fLT/iFhYW67bbbTvo8brdbhw8f9rm53W7/qw8QKY+myuPx6IYrLtHA3vH699vzlDhgkIKs/l6RM8iqZYuV0G+gQkKcZpdiSU2j6+jp4d30l2mr5D5aWeV4fpFbw19YoUHxzbVv7k3a8+qf5AqvrfXf56vS4zGhYkg/bf57euL98ng8uuvev5tdDmqQX+E/YcIEdejQQfv371dmZqYiIiLUq1cvZWVl+fWiqampcrlcPrdpU57x6zkCSdNmzfXCzLn6z2dfacH7yzRjzj917NgxNW7SzOzSIClz03rt2/Oj+g5k5H+6dGkVrUb1wrQq9QoVvDVMBW8NU+9zYzXy0vYqeGuYghwOffLtPnW+Z5Fa//Vttbpjge5M+0JNouroh9xis8u3pZ+Cf4L25+7TE8/PoOs3CKrGWyDya8Pfl19+qY8//lgNGjRQgwYNtHjxYt11113q3bu3Pv30U4WHh5/S86SkpCg5OdlnLa/szO+Sw8LqKCysjooOF2pt+pf66+hxZpcESSv/u1gt27RTXOtzzC7FslZs2qfu973vszbjbz21PbtQU97b7NPdFxT9NOXrc16sGkaG6sOM3TVaK34J/uw9WXpq6iuKdNUzu6SA4wjUeX018Sv8y8rKFBz8y0McDodmzJih0aNHq2/fvpo3b94pPY/T6ZTT6Tt+Lao84k8pAWVt+hfyeDxq3qKl9u7O0ssvPa+4Fq106ZWDzS7N0srLSpWbvcd7/0Butn78brvqRkQqulGsJKmstFhrVi3XjX+5x6wybaG4/Ji27jnks1biPqaCIrd3/ea+Z2n73kLlFZXrwrMb6unhFyrtwy3aue9wzRdscWWlpdq395cfqnL37dX3OzJVNzJSUdENNPnh8fpu+zY98vRUVVZU6mD+T/uT6ka6VLt2bbPKRg3yK/zbtWundevWqX379j7r06ZNkyRddZU9x6olxUWaNX2q8vbnKiLSpd79kjTib3crOJh/RKfTrh1bNfmBu7z3/znrBUnSRUmX647kRyRJ6SuWSfKox8UDTKgQv3Z2E5cevfEC1a8boqwDJXp20bdK+3Cr2WVZ0s7MLfr7PXd478+e9pwkKfHSK3XTbSP11RcrJEl33/4nn8c9NXWWOv78lmW7s3bfLzk8nlPfbZOamqpVq1bpww8/PO7xu+66SzNnzlRlZdUNPyez99CZ2/lbze78UrNLwM8GPPj+yU9CjVg39TqzS8CvnBNT57Q+/5sZe05+0ikaFh94+7/82ouQkpJywuCXpOnTp/+u4AcAIJA4qvEWiAJ1IyIAADhN+HhfAAAMLL7Zn/AHAMDI6m/1Y+wPAIDN0PkDAGBg9c6Y8AcAwICxPwAAsBQ6fwAADKzd9xP+AABUwdgfAABYCp0/AAAGVu+MCX8AAAysPvYn/AEAMLB29Ft/sgEAAAzo/AEAMLD41J/wBwDAKMjig3/G/gAA2AzhDwCAgcNRfTd/pKamqlu3boqIiFCjRo00ePBgZWZm+pxTXl6uUaNGKTo6WnXr1tWQIUOUm5vr1+sQ/gAAGDiq8T9/rFixQqNGjVJ6erqWLVumo0ePasCAASopKfGeM27cOC1evFgLFy7UihUrlJ2drWuvvdav1+GaPwAAAWLJkiU+9+fOnatGjRopIyNDffr0UWFhoWbPnq158+YpMTFRkjRnzhy1b99e6enp6tGjxym9Dp0/AAAG1Tn2d7vdOnz4sM/N7XafUh2FhYWSpKioKElSRkaGjh49qqSkJO857dq1U1xcnFavXn3KXx/hDwCAQZAc1XZLTU2Vy+XyuaWmpp60hsrKSo0dO1a9evVShw4dJEk5OTkKCQlRvXr1fM6NiYlRTk7OKX99jP0BADiNUlJSlJyc7LPmdDpP+rhRo0Zp06ZN+vzzz6u9JsIfAACD6vyQH6fTeUph/2ujR4/WBx98oJUrV6pZs2be9djYWB05ckSHDh3y6f5zc3MVGxt7ys/P2B8AAAOz3urn8Xg0evRoLVq0SJ988olatWrlczw+Pl61a9fW8uXLvWuZmZnKyspSQkLCKb8OnT8AAAb+vkWvuowaNUrz5s3Te++9p4iICO91fJfLpbCwMLlcLo0YMULJycmKiopSZGSkxowZo4SEhFPe6S8R/gAABIwZM2ZIki6++GKf9Tlz5ujWW2+VJE2ZMkVBQUEaMmSI3G63Bg4cqOnTp/v1OoQ/AAAGQSZ9tL/H4znpOaGhoUpLS1NaWtrvfh3CHwAAA7PG/jWFDX8AANgMnT8AAAbV+Va/QET4AwBgwNgfAABYCp0/AAAGZu32rymEPwAABoz9AQCApdD5AwBgwG5/AABsxuLZT/gDAGAUZPHWn2v+AADYTMB0/vXq1Da7BPysojLM7BLwsxVPX2N2CfhZ50vvN7sE/ErZ+mmn9fmt3fcHUPgDABAwLJ7+jP0BALAZOn8AAAys/iE/hD8AAAYW3+zP2B8AALuh8wcAwMDijT/hDwBAFRZPf8b+AADYDJ0/AAAG7PYHAMBmrL7bn/AHAMDA4tnPNX8AAOyGzh8AACOLt/6EPwAABlbf8MfYHwAAm6HzBwDAgN3+AADYjMWzn7E/AAB2Q+cPAICRxVt/wh8AAAN2+wMAAEuh8wcAwIDd/gAA2IzFs5/wBwCgCounP9f8AQCwGTp/AAAMrL7bn/AHAMDA6hv+GPsDAGAzdP4AABhYvPEn/AEAqMLi6c/YHwAAm6HzBwDAgN3+AADYDLv9AQCApdD5AwBgYPHGn/AHAKAKi6c/4Q8AgIHVN/xxzR8AAJuh8wcAwMDqu/0J/z9o5vSX9MqMNJ+1li1b6d+LPzKpIvv4dv06LXxrrrZnblVB3gE9OvkF9eqb6D3u8Xj02qzp+uj9f6m4qEjndeqiu+9/SM2atzCxamva+u3XWrzwDe3asVUHC/J078R/qFuvi73Hpz/7qFYu+8DnMZ27JijlqZdquFLre/Cvl+mhkZf5rGXuylGXaycprnGUMj98/LiPu3n8bP374/U1UeIZweLZT/hXh7PanK0Zs1713q9Vi7/WmlBeXqbWZ7fVwCuu0WMp46ocX/DmHL27cJ7uf3iSYps01dxXpill7EjNnveuQpxOEyq2rvLyMrVofbYuHniVnn98/HHP6dy1p/523yPe+8G1Q2qqPNvZvDNbl4/85QerYxWVkqQ9uQfVMinF59zbh/TSuD8naekXm2u0RpiLlKoGtWrVUoMGDc0uw3YuTOitCxN6H/eYx+PRogVv6uZb71DPPv0kSRMeeVLXX95PX6z8RP36D6rJUi3v/At76fwLe/3mObVr11a9qAY1VJG9HauoVG5+UZX1ykpPlfWr+nXWv5Z9rZKyIzVV3pnB4q0/4V8NsrJ+1IDE3nKGONWpcxeNHpusxo2bmF2WreVk71VBfp7O79bDuxZeN0Ltzu2oLZu+IfxNsOXbDN15fX+FR0TovC7dNPTWvykisp7ZZVlSm7iG+v6/T6rcfVRffbtLj7z0vnbnHKxy3vntm6tLu+YaN/ltE6oMbFbf7U/4/0EdO3bWY0+kqkXLVsrL269XZqRpxPBhWrjofYWH1zW7PNsqyM+TJNWPivZZrx8VrYP5+WaUZGtduibowov6qVFsU+Vm79H8OWma/ODdeuKFOQqqVcvs8ixl7aYfdOcjb2r7j7mKbeDSg38dpI9fHaf4655Ucanb59zhgxO09ft9Sv9ml0nVwix+h//WrVuVnp6uhIQEtWvXTtu2bdPUqVPldrs1bNgwJSYmnvQ53G633G7f/wmPOULkPAOvw/bq3cf753PatlXHjp11+cBELVu6RIOvvc7EyoDA0bPfQO+f41q1UVzrNrpn+GBt/jZDHc+/0MTKrOe/X2zx/nnTjmyt3fiDMj98XEMGXKDX3l3tPRbqrK2hg7pq8qwlZpQZ8Ky+29+v9/kvWbJEXbp00X333afzzz9fS5YsUZ8+fbRz5079+OOPGjBggD755JOTPk9qaqpcLpfP7R/PpP7uLyKQRERGKq5FS+3O+tHsUmwtKvqna8sHC3y7/IMF+aofHX28h6AGxTRupghXPeXu3W12KZZXWFymnVn7dVZz331J1yR1UZ3QEL31wRqTKgtsjmq8+WPlypW68sor1aRJEzkcDr377rs+xz0ejx555BE1btxYYWFhSkpK0o4dO/z++vwK/8cff1zjx49Xfn6+5syZo5tuukl33HGHli1bpuXLl2v8+PGaPHnySZ8nJSVFhYWFPrf77k856ePOBKWlJdqze7caNGQDoJlimzRVVHQDrV/3lXetpKRY27Zs1LkdOptYGSQp/0Cuig8Xql40GwBPt/CwELVq1kA5eYU+67cO7qn/rNiovIPFJlWG4ykpKVHnzp2VlpZ23OPPPPOMXnzxRc2cOVNfffWVwsPDNXDgQJWXl/v1On6N/Tdv3qzXX39dknTDDTfolltu0XXX/TLavvnmmzVnzpyTPo/T6awy4i854vGnlIAx5R9Pq0/ffmrcpIkOHNivmWnTFFQrSJcOusLs0iyvrLRUe/dkee/nZO/Vzu3bFBnpUqPYxrpm6DDNm/uKmjaPU+PGTTV3VpqiGzRUrz4nvzQF/5SXlSon+5cufn/OXv3wXabqRrhUNyJS77wxS917J8pVP1q5+/Zo3qwXFdOkuTrHJ5hYtTWljrtG/1m5UVnZBWrSyKWHRl6uispKvb0kw3tO6+YNdNEFZ2nwmBkmVhrgTBr7Dxo0SIMGHX9Dssfj0QsvvKCHHnpIV199tSTp9ddfV0xMjN5991396U9/OuXX8fuav+PnCyFBQUEKDQ2Vy+XyHouIiFBhYeGJHmpJubm5SplwrwoPHVL9+lHqckG8XntrgepHRZldmuVt37ZZ940a4b0/88VnJUn9L7tK9z88SUOH3abysjK9MPlxFRcXqUOn85U6ZQbv8T8Nvtu+RU+MH+m9/8bLUyRJffpfob/c/YCydu3QymUfqKSkSPWjG6rTBT10w60jVTuE9/pXt6Yx9fR66m2KctVR3sFifbnhe/X983M+Hf7wqxO0N/eQPl69zcRKA1t17vY/3j634zXBJ7Nr1y7l5OQoKSnJu+ZyudS9e3etXr3ar/B3eDyeU265O3furKefflqXXnqpJGnTpk1q166dgoN/+hli1apVGj58uL7//vtTLuB/ztTO34ryi3m/b6DIL+J7ESh6DrbGpUmrKFs/7bQ+f1aB++QnnaJXX0zVY4895rM2ceJEPfroo7/5OIfDoUWLFmnw4MGSpC+//FK9evVSdna2Gjdu7D3vhhtukMPh0IIFC065Jr86/7/97W+qqKjw3u/QoYPP8Y8++uiUdvsDAGAXKSkpSk5O9lkz+91tfoX/yJEjf/P4U0899YeKAQAgEFTnJf/fM+I/ntjYWEk/XW7+deefm5urLl26+PVc/EpfAAAMHI7qu1WXVq1aKTY2VsuXL/euHT58WF999ZUSEvzbPMsn/AEAECCKi4u1c+dO7/1du3Zpw4YNioqKUlxcnMaOHatJkybp7LPPVqtWrfTwww+rSZMm3n0Bp4rwBwCgCnPe67du3Tr169fPe/9/ewWGDx+uuXPn6v7771dJSYnuvPNOHTp0SBdddJGWLFmi0NBQv17Hr93+pxO7/QMHu/0DB7v9Awe7/QPL6d7tv/dQ9f3ba1ov8N7SyjV/AABshrE/AAAGFv+9PoQ/AABG/FY/AABgKXT+AAAYVOdn+wciwh8AACNrZz/hDwCAkcWzn2v+AADYDZ0/AAAGVt/tT/gDAGBg9Q1/jP0BALAZOn8AAIys3fgT/gAAGFk8+xn7AwBgN3T+AAAYsNsfAACbYbc/AACwFDp/AAAMrD72p/MHAMBm6PwBADCg8wcAAJZC5w8AgIHVd/sT/gAAGDD2BwAAlkLnDwCAgcUbf8IfAIAqLJ7+jP0BALAZOn8AAAzY7Q8AgM2w2x8AAFgKnT8AAAYWb/wJfwAAqrB4+hP+AAAYWH3DH9f8AQCwGTp/AAAMrL7b3+HxeDxmF2EFbrdbqampSklJkdPpNLsc2+P7ETj4XgQOvhf4H8K/mhw+fFgul0uFhYWKjIw0uxzb4/sROPheBA6+F/gfrvkDAGAzhD8AADZD+AMAYDOEfzVxOp2aOHEim2gCBN+PwMH3InDwvcD/sOEPAACbofMHAMBmCH8AAGyG8AcAwGYIfwAAbIbwryZpaWlq2bKlQkND1b17d61Zs8bskmxp5cqVuvLKK9WkSRM5HA69++67ZpdkS6mpqerWrZsiIiLUqFEjDR48WJmZmWaXZVszZsxQp06dFBkZqcjISCUkJOijjz4yuyyYiPCvBgsWLFBycrImTpyor7/+Wp07d9bAgQO1f/9+s0uznZKSEnXu3FlpaWlml2JrK1as0KhRo5Senq5ly5bp6NGjGjBggEpKSswuzZaaNWumyZMnKyMjQ+vWrVNiYqKuvvpqbd682ezSYBLe6lcNunfvrm7dumnatGmSpMrKSjVv3lxjxozRAw88YHJ19uVwOLRo0SINHjzY7FJs78CBA2rUqJFWrFihPn36mF0OJEVFRenZZ5/ViBEjzC4FJqDz/4OOHDmijIwMJSUledeCgoKUlJSk1atXm1gZEDgKCwsl/RQ4MFdFRYXmz5+vkpISJSQkmF0OTBJsdgFnury8PFVUVCgmJsZnPSYmRtu2bTOpKiBwVFZWauzYserVq5c6dOhgdjm2tXHjRiUkJKi8vFx169bVokWLdO6555pdFkxC+AM4rUaNGqVNmzbp888/N7sUW2vbtq02bNigwsJCvfPOOxo+fLhWrFjBDwA2Rfj/QQ0aNFCtWrWUm5vrs56bm6vY2FiTqgICw+jRo/XBBx9o5cqVatasmdnl2FpISIjatGkjSYqPj9fatWs1depUvfzyyyZXBjNwzf8PCgkJUXx8vJYvX+5dq6ys1PLly7meBtvyeDwaPXq0Fi1apE8++UStWrUyuyQYVFZWyu12m10GTELnXw2Sk5M1fPhwde3aVRdeeKFeeOEFlZSU6LbbbjO7NNspLi7Wzp07vfd37dqlDRs2KCoqSnFxcSZWZi+jRo3SvHnz9N577ykiIkI5OTmSJJfLpbCwMJOrs5+UlBQNGjRIcXFxKioq0rx58/TZZ59p6dKlZpcGk/BWv2oybdo0Pfvss8rJyVGXLl304osvqnv37maXZTufffaZ+vXrV2V9+PDhmjt3bs0XZFMOh+O463PmzNGtt95as8VAI0aM0PLly7Vv3z65XC516tRJEyZMUP/+/c0uDSYh/AEAsBmu+QMAYDOEPwAANkP4AwBgM4Q/AAA2Q/gDAGAzhD8AADZD+AMAYDOEPwAANkP4AwBgM4Q/AAA2Q/gDAGAzhD8AADbz/8y+/Nsl2UmlAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy: 0.5488505747126436\n",
      "precision: 0.5446881684577992\n"
     ]
    }
   ],
   "source": [
    "model = MIDfinal2(values_train)\n",
    "model.eval()\n",
    "y_true = []\n",
    "y_pred = []\n",
    "test_dataset = ImageDataset(data_dir, test_patients, test_images_df)\n",
    "for i in tqdm(range(len(test_dataset))):\n",
    "    image, label, path = test_dataset[i]\n",
    "    image = image.unsqueeze(0)\n",
    "    image = image.to(device)\n",
    "    output = model(image)\n",
    "    binary = output[0]\n",
    "    resnet = output[1]\n",
    "    output = output[2]\n",
    "    y_true.append(label)\n",
    "    y_pred.append(output)\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "plt.show()\n",
    "\n",
    "print(\"accuracy:\", accuracy_score(y_true, y_pred))\n",
    "print(\"precision:\", precision_score(y_true, y_pred, average=\"macro\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}