FusionModels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run C:/Users/MohammedSB/Desktop/projects/Hypertension/Requirements.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Seed\n",
    "set_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = r\"D:\\\\MohammedSB\\\\Fundus\\\\\"\n",
    "CSV_PATH = {\"HTNPath\": PATH + r\"HTN\", \"NonHTNPath\": PATH + \"NonHTN\"}\n",
    "\n",
    "MODELS_PATH = r\"C:\\Users\\MohammedSB\\Desktop\\projects\\Hypertension\\Models\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set, test_set = get_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_transform = T.Compose([\n",
    "    T.Resize((500, 500)),\n",
    "    T.ToTensor(),\n",
    "    T.RandomHorizontalFlip(0.5),\n",
    "    T.RandomRotation(degrees=(0, 360)),\n",
    "    T.GaussianBlur(5),\n",
    "    T.Normalize(mean=[0.3675, 0.1760, 0.1047], std=[0.2289, 0.1180, 0.0748]),\n",
    "])\n",
    "\n",
    "test_transform = T.Compose([\n",
    "    T.Resize((500, 500)),\n",
    "    T.ToTensor(),\n",
    "    T.Normalize(mean=[0.3675, 0.1760, 0.1047], std=[0.2289, 0.1180, 0.0748]),\n",
    "])\n",
    "\n",
    "train_dataset = HypertensionDataset(CSV_PATH, train_set, train_transform=train_transform)\n",
    "test_dataset = HypertensionDataset(CSV_PATH, test_set, test_transform=test_transform)\n",
    "\n",
    "train_loader, test_loader = train_test_loader(train_dataset, test_dataset, batch_size=8)\n",
    "\n",
    "# val_dataset = HypertensionDataset(CSV_PATH, val_set, test_transform=test_transform)\n",
    "# val_loader = DataLoader(val_dataset, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "FM_PATH = MODELS_PATH + r\"\\FundusModel.pth\"\n",
    "DM_PATH = MODELS_PATH + r\"\\DemographicFCNN.pth\"\n",
    "\n",
    "criterion = nn.BCEWithLogitsLoss()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# FeatureFusion (Joint Fusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabular_model = nn.Sequential(\n",
    "    nn.Linear(in_features=2, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=32),\n",
    ")\n",
    "\n",
    "image_model = get_densenet201(device=device, freeze=True, with_mlp=False, outputs=32)\n",
    "\n",
    "fusion_model = nn.Sequential(\n",
    "    nn.Linear(in_features=64, out_features=128),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(),\n",
    "    nn.Linear(in_features=128, out_features=32),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(),\n",
    "    nn.Linear(in_features=32, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(),\n",
    "    nn.Linear(in_features=16, out_features=1),\n",
    ")\n",
    "\n",
    "tabular_model, image_model, fusion_model = tabular_model.to(device).float(), image_model.to(device).float(),\\\n",
    "                                                fusion_model.to(device).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    \"image_model\": image_model,\n",
    "    \"tabular_model\": tabular_model,\n",
    "    \"fusion_model\": fusion_model,\n",
    "}\n",
    "\n",
    "early_stop = {\n",
    "    \"patience\": 5,\n",
    "    \"min_delta\": 0.001,\n",
    "    \"multip\": 5,\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': image_model.parameters(), 'lr': 1e-4},\n",
    "    {'params': tabular_model.parameters(), 'lr': 1e-4},\n",
    "    {'params': fusion_model.parameters(), 'lr': 1e-4}],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beginning Training: \n",
      "\n",
      "Epoch 1/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:52<00:00,  2.21it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.702299,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.13it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.697782,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 2/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.700149,    Accuracy: 49.29%,    Correct Counter: 451/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  0.99 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.11it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.695274,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 3/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.699047,    Accuracy: 49.95%,    Correct Counter: 457/915,    F1 Score: 0.64,    Precision: 0.49,    Recall:  0.93 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.09it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.693218,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 4/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.29it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.698067,    Accuracy: 49.73%,    Correct Counter: 455/915,    F1 Score: 0.63,    Precision: 0.49,    Recall:  0.88 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.09it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.692038,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 5/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.695249,    Accuracy: 52.57%,    Correct Counter: 481/915,    F1 Score: 0.65,    Precision: 0.51,    Recall:  0.92 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:06<00:00,  4.17it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.690046,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 6/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.694403,    Accuracy: 52.35%,    Correct Counter: 479/915,    F1 Score: 0.64,    Precision: 0.51,    Recall:  0.88 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.685328,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 7/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.688128,    Accuracy: 54.10%,    Correct Counter: 495/915,    F1 Score: 0.64,    Precision: 0.52,    Recall:  0.85 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.05it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.678400,    Accuracy: 62.28%,    Correct Counter: 142/228,    F1 Score: 0.72,    Precision: 0.57,    Recall:  0.97 \n",
      "\n",
      "Epoch 8/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.681669,    Accuracy: 55.19%,    Correct Counter: 505/915,    F1 Score: 0.65,    Precision: 0.52,    Recall:  0.84 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.665225,    Accuracy: 65.79%,    Correct Counter: 150/228,    F1 Score: 0.73,    Precision: 0.60,    Recall:  0.93 \n",
      "\n",
      "Epoch 9/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.671140,    Accuracy: 58.14%,    Correct Counter: 532/915,    F1 Score: 0.66,    Precision: 0.54,    Recall:  0.86 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.650504,    Accuracy: 65.79%,    Correct Counter: 150/228,    F1 Score: 0.73,    Precision: 0.60,    Recall:  0.93 \n",
      "\n",
      "Epoch 10/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.643410,    Accuracy: 63.83%,    Correct Counter: 584/915,    F1 Score: 0.69,    Precision: 0.59,    Recall:  0.83 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.14it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.628206,    Accuracy: 66.67%,    Correct Counter: 152/228,    F1 Score: 0.73,    Precision: 0.61,    Recall:  0.90 \n",
      "\n",
      "Epoch 11/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.631817,    Accuracy: 64.81%,    Correct Counter: 593/915,    F1 Score: 0.69,    Precision: 0.60,    Recall:  0.80 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.11it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.610490,    Accuracy: 65.35%,    Correct Counter: 149/228,    F1 Score: 0.71,    Precision: 0.60,    Recall:  0.87 \n",
      "\n",
      "Epoch 12/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.622791,    Accuracy: 66.78%,    Correct Counter: 611/915,    F1 Score: 0.71,    Precision: 0.61,    Recall:  0.85 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.14it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.603864,    Accuracy: 66.67%,    Correct Counter: 152/228,    F1 Score: 0.73,    Precision: 0.61,    Recall:  0.89 \n",
      "\n",
      "Epoch 13/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.32it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.599170,    Accuracy: 68.74%,    Correct Counter: 629/915,    F1 Score: 0.72,    Precision: 0.64,    Recall:  0.82 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.11it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.590259,    Accuracy: 67.98%,    Correct Counter: 155/228,    F1 Score: 0.73,    Precision: 0.63,    Recall:  0.87 \n",
      "\n",
      "Epoch 14/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.600270,    Accuracy: 68.09%,    Correct Counter: 623/915,    F1 Score: 0.71,    Precision: 0.63,    Recall:  0.81 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.587897,    Accuracy: 66.67%,    Correct Counter: 152/228,    F1 Score: 0.72,    Precision: 0.62,    Recall:  0.87 \n",
      "\n",
      "Epoch 15/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.593490,    Accuracy: 68.52%,    Correct Counter: 627/915,    F1 Score: 0.72,    Precision: 0.63,    Recall:  0.85 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.590036,    Accuracy: 65.79%,    Correct Counter: 150/228,    F1 Score: 0.72,    Precision: 0.61,    Recall:  0.88 \n",
      "\n",
      "Epoch 16/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.574045,    Accuracy: 69.95%,    Correct Counter: 640/915,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.86 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.11it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.588508,    Accuracy: 67.11%,    Correct Counter: 153/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.88 \n",
      "\n",
      "Epoch 17/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.32it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.571185,    Accuracy: 69.62%,    Correct Counter: 637/915,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.86 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:06<00:00,  4.15it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.592587,    Accuracy: 66.23%,    Correct Counter: 151/228,    F1 Score: 0.72,    Precision: 0.61,    Recall:  0.88 \n",
      "\n",
      "Epoch 18/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.576748,    Accuracy: 70.93%,    Correct Counter: 649/915,    F1 Score: 0.75,    Precision: 0.64,    Recall:  0.89 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.08it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.593682,    Accuracy: 67.11%,    Correct Counter: 153/228,    F1 Score: 0.73,    Precision: 0.61,    Recall:  0.90 \n",
      "\n",
      "Epoch 19/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.32it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.564349,    Accuracy: 71.04%,    Correct Counter: 650/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.89 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.594221,    Accuracy: 65.79%,    Correct Counter: 150/228,    F1 Score: 0.72,    Precision: 0.61,    Recall:  0.88 \n",
      "\n",
      "Epoch 20/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.582065,    Accuracy: 70.27%,    Correct Counter: 643/915,    F1 Score: 0.75,    Precision: 0.64,    Recall:  0.91 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.595476,    Accuracy: 65.79%,    Correct Counter: 150/228,    F1 Score: 0.72,    Precision: 0.61,    Recall:  0.88 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs = 200\n",
    "metrics = train_val(epochs=epochs, models=models, criterion=criterion, optimizer=optimizer, train_loader=train_loader,\n",
    "                    val_loader=test_loader, device=device, early_stop=early_stop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(image_model.state_dict(), MODELS_PATH+\"\\FeatureFusion\\FundusPath.pth\")\n",
    "torch.save(tabular_model.state_dict(), MODELS_PATH+\"\\FeatureFusion\\DemographicPath.pth\")\n",
    "torch.save(fusion_model.state_dict(), MODELS_PATH+\"\\FeatureFusion\\FusionPath.pth\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PredictionFusion (Joint Fusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tabular_model = nn.Sequential(\n",
    "    nn.Linear(in_features=2, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=32),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=32, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=16, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=1),\n",
    ")\n",
    "\n",
    "image_model = get_densenet201(device=device, freeze=True, with_mlp=True, outputs=1)\n",
    "\n",
    "fusion_model = nn.Sequential(\n",
    "    nn.Linear(in_features=2, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=16, out_features=4),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=4, out_features=1),\n",
    ")\n",
    "\n",
    "tabular_model, image_model, fusion_model = tabular_model.to(device).float(), image_model.to(device).float(),\\\n",
    "                                                fusion_model.to(device).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beginning Training: \n",
      "\n",
      "Epoch 1/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.27it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.693080,    Accuracy: 47.32%,    Correct Counter: 433/915,    F1 Score: 0.61,    Precision: 0.48,    Recall:  0.85 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.09it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.691628,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  0.99 \n",
      "\n",
      "Epoch 2/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.692121,    Accuracy: 54.10%,    Correct Counter: 495/915,    F1 Score: 0.59,    Precision: 0.52,    Recall:  0.68 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.689809,    Accuracy: 57.02%,    Correct Counter: 130/228,    F1 Score: 0.69,    Precision: 0.54,    Recall:  0.98 \n",
      "\n",
      "Epoch 3/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.691561,    Accuracy: 54.97%,    Correct Counter: 503/915,    F1 Score: 0.60,    Precision: 0.53,    Recall:  0.68 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.689015,    Accuracy: 60.09%,    Correct Counter: 137/228,    F1 Score: 0.69,    Precision: 0.56,    Recall:  0.90 \n",
      "\n",
      "Epoch 4/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.28it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.689727,    Accuracy: 60.11%,    Correct Counter: 550/915,    F1 Score: 0.65,    Precision: 0.57,    Recall:  0.76 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.687850,    Accuracy: 57.46%,    Correct Counter: 131/228,    F1 Score: 0.69,    Precision: 0.54,    Recall:  0.96 \n",
      "\n",
      "Epoch 5/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.687515,    Accuracy: 64.37%,    Correct Counter: 589/915,    F1 Score: 0.69,    Precision: 0.60,    Recall:  0.83 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.687808,    Accuracy: 58.77%,    Correct Counter: 134/228,    F1 Score: 0.70,    Precision: 0.55,    Recall:  0.96 \n",
      "\n",
      "Epoch 6/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.686768,    Accuracy: 64.59%,    Correct Counter: 591/915,    F1 Score: 0.69,    Precision: 0.60,    Recall:  0.80 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.14it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.685283,    Accuracy: 65.79%,    Correct Counter: 150/228,    F1 Score: 0.71,    Precision: 0.61,    Recall:  0.83 \n",
      "\n",
      "Epoch 7/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.686351,    Accuracy: 62.62%,    Correct Counter: 573/915,    F1 Score: 0.62,    Precision: 0.61,    Recall:  0.64 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.09it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.684772,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.71,    Precision: 0.60,    Recall:  0.88 \n",
      "\n",
      "Epoch 8/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.684649,    Accuracy: 64.92%,    Correct Counter: 594/915,    F1 Score: 0.66,    Precision: 0.62,    Recall:  0.71 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.682796,    Accuracy: 68.42%,    Correct Counter: 156/228,    F1 Score: 0.70,    Precision: 0.66,    Recall:  0.75 \n",
      "\n",
      "Epoch 9/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.686273,    Accuracy: 60.87%,    Correct Counter: 557/915,    F1 Score: 0.59,    Precision: 0.60,    Recall:  0.57 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.686077,    Accuracy: 61.40%,    Correct Counter: 140/228,    F1 Score: 0.71,    Precision: 0.57,    Recall:  0.96 \n",
      "\n",
      "Epoch 10/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.683453,    Accuracy: 63.72%,    Correct Counter: 583/915,    F1 Score: 0.65,    Precision: 0.61,    Recall:  0.70 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.682634,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 11/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.681570,    Accuracy: 63.72%,    Correct Counter: 583/915,    F1 Score: 0.65,    Precision: 0.61,    Recall:  0.70 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.14it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.681410,    Accuracy: 68.42%,    Correct Counter: 156/228,    F1 Score: 0.74,    Precision: 0.63,    Recall:  0.90 \n",
      "\n",
      "Epoch 12/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.679912,    Accuracy: 66.34%,    Correct Counter: 607/915,    F1 Score: 0.63,    Precision: 0.68,    Recall:  0.59 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.683114,    Accuracy: 63.16%,    Correct Counter: 144/228,    F1 Score: 0.71,    Precision: 0.58,    Recall:  0.93 \n",
      "\n",
      "Epoch 13/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.678954,    Accuracy: 65.57%,    Correct Counter: 600/915,    F1 Score: 0.66,    Precision: 0.63,    Recall:  0.68 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.13it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.679668,    Accuracy: 66.23%,    Correct Counter: 151/228,    F1 Score: 0.72,    Precision: 0.61,    Recall:  0.89 \n",
      "\n",
      "Epoch 14/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.675448,    Accuracy: 68.09%,    Correct Counter: 623/915,    F1 Score: 0.67,    Precision: 0.67,    Recall:  0.68 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.07it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.677936,    Accuracy: 66.67%,    Correct Counter: 152/228,    F1 Score: 0.72,    Precision: 0.61,    Recall:  0.88 \n",
      "\n",
      "Epoch 15/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.674766,    Accuracy: 65.25%,    Correct Counter: 597/915,    F1 Score: 0.66,    Precision: 0.63,    Recall:  0.71 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.13it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.680907,    Accuracy: 61.84%,    Correct Counter: 141/228,    F1 Score: 0.71,    Precision: 0.57,    Recall:  0.95 \n",
      "\n",
      "Epoch 16/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.32it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.672729,    Accuracy: 66.56%,    Correct Counter: 609/915,    F1 Score: 0.68,    Precision: 0.64,    Recall:  0.72 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.13it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.677431,    Accuracy: 64.04%,    Correct Counter: 146/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.93 \n",
      "\n",
      "Epoch 17/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.673093,    Accuracy: 63.93%,    Correct Counter: 585/915,    F1 Score: 0.66,    Precision: 0.61,    Recall:  0.73 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.679789,    Accuracy: 61.40%,    Correct Counter: 140/228,    F1 Score: 0.71,    Precision: 0.56,    Recall:  0.96 \n",
      "\n",
      "Epoch 18/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.669842,    Accuracy: 65.03%,    Correct Counter: 595/915,    F1 Score: 0.66,    Precision: 0.62,    Recall:  0.71 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.05it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.666181,    Accuracy: 67.54%,    Correct Counter: 154/228,    F1 Score: 0.72,    Precision: 0.63,    Recall:  0.83 \n",
      "\n",
      "Epoch 19/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.660101,    Accuracy: 67.32%,    Correct Counter: 616/915,    F1 Score: 0.68,    Precision: 0.65,    Recall:  0.70 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.08it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.650850,    Accuracy: 68.42%,    Correct Counter: 156/228,    F1 Score: 0.68,    Precision: 0.68,    Recall:  0.68 \n",
      "\n",
      "Epoch 20/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.644793,    Accuracy: 66.12%,    Correct Counter: 605/915,    F1 Score: 0.65,    Precision: 0.66,    Recall:  0.64 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.636567,    Accuracy: 70.18%,    Correct Counter: 160/228,    F1 Score: 0.74,    Precision: 0.65,    Recall:  0.86 \n",
      "\n",
      "Epoch 21/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.626398,    Accuracy: 66.67%,    Correct Counter: 610/915,    F1 Score: 0.68,    Precision: 0.64,    Recall:  0.72 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.10it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.633072,    Accuracy: 68.42%,    Correct Counter: 156/228,    F1 Score: 0.74,    Precision: 0.63,    Recall:  0.89 \n",
      "\n",
      "Epoch 22/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.29it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.619394,    Accuracy: 65.68%,    Correct Counter: 601/915,    F1 Score: 0.69,    Precision: 0.61,    Recall:  0.79 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.660902,    Accuracy: 59.65%,    Correct Counter: 136/228,    F1 Score: 0.71,    Precision: 0.55,    Recall:  0.98 \n",
      "\n",
      "Epoch 23/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.614469,    Accuracy: 66.56%,    Correct Counter: 609/915,    F1 Score: 0.71,    Precision: 0.61,    Recall:  0.86 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.07it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.647369,    Accuracy: 61.40%,    Correct Counter: 140/228,    F1 Score: 0.71,    Precision: 0.57,    Recall:  0.95 \n",
      "\n",
      "Epoch 24/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.29it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.615028,    Accuracy: 67.43%,    Correct Counter: 617/915,    F1 Score: 0.71,    Precision: 0.62,    Recall:  0.83 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.14it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.643888,    Accuracy: 62.72%,    Correct Counter: 143/228,    F1 Score: 0.72,    Precision: 0.58,    Recall:  0.95 \n",
      "\n",
      "Epoch 25/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.605743,    Accuracy: 68.42%,    Correct Counter: 626/915,    F1 Score: 0.72,    Precision: 0.63,    Recall:  0.83 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.644657,    Accuracy: 62.28%,    Correct Counter: 142/228,    F1 Score: 0.71,    Precision: 0.57,    Recall:  0.95 \n",
      "\n",
      "Epoch 26/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.601183,    Accuracy: 68.63%,    Correct Counter: 628/915,    F1 Score: 0.72,    Precision: 0.64,    Recall:  0.82 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.657703,    Accuracy: 59.21%,    Correct Counter: 135/228,    F1 Score: 0.70,    Precision: 0.55,    Recall:  0.98 \n",
      "\n",
      "Epoch 27/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.29it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.599380,    Accuracy: 69.29%,    Correct Counter: 634/915,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.86 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.13it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.637587,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.93 \n",
      "\n",
      "Epoch 28/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:50<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.603993,    Accuracy: 68.52%,    Correct Counter: 627/915,    F1 Score: 0.72,    Precision: 0.63,    Recall:  0.84 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.06it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.650862,    Accuracy: 60.96%,    Correct Counter: 139/228,    F1 Score: 0.71,    Precision: 0.56,    Recall:  0.96 \n",
      "\n",
      "Epoch 29/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.31it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.598367,    Accuracy: 67.87%,    Correct Counter: 621/915,    F1 Score: 0.72,    Precision: 0.62,    Recall:  0.87 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.14it/s]\n",
      "Batch in Progress:   0%|                                                    | 0/115 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.644455,    Accuracy: 62.28%,    Correct Counter: 142/228,    F1 Score: 0.71,    Precision: 0.57,    Recall:  0.95 \n",
      "\n",
      "Epoch 30/200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|██████████████████████████████████████████| 115/115 [00:49<00:00,  2.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.595121,    Accuracy: 68.74%,    Correct Counter: 629/915,    F1 Score: 0.72,    Precision: 0.63,    Recall:  0.84 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|████████████████████████████████████████████| 29/29 [00:07<00:00,  4.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.653787,    Accuracy: 60.53%,    Correct Counter: 138/228,    F1 Score: 0.71,    Precision: 0.56,    Recall:  0.98 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "models = {\n",
    "    \"image_model\": image_model,\n",
    "    \"tabular_model\": tabular_model,\n",
    "    \"fusion_model\": fusion_model,\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': image_model.parameters(), 'lr': 1e-4},\n",
    "    {'params': tabular_model.parameters(), 'lr': 1e-4},\n",
    "    {'params': fusion_model.parameters(), 'lr': 1e-4}],\n",
    ")\n",
    "\n",
    "epochs = 200\n",
    "metrics = train_val(epochs=epochs, models=models, criterion=criterion, optimizer=optimizer, train_loader=train_loader,\n",
    "                    val_loader=test_loader, device=device, early_stop=early_stop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(image_model.state_dict(), MODELS_PATH + \"\\PredictionFusion\\FundusPath.pth\")\n",
    "torch.save(tabular_model.state_dict(), MODELS_PATH + \"\\PredictionFusion\\DemographicPath.pth\")\n",
    "torch.save(fusion_model.state_dict(), MODELS_PATH + \"\\PredictionFusion\\FusionPath.pth\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LateFusion (Late Fusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_model = get_densenet201(device=device, freeze=True, with_mlp=True, outputs=1)\n",
    "image_model.load_state_dict(torch.load(FM_PATH))\n",
    "\n",
    "image_model = image_model.to(device).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    \"image_model\": image_model,\n",
    "    \"tabular_model\": None,\n",
    "    \"fusion_model\": None,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y = build_tabular_dataset(models, train_dataset, method=\"keep_features\")\n",
    "test_x, test_y = build_tabular_dataset(models, test_dataset, method=\"keep_features\")\n",
    "\n",
    "train_fusion_set = InputOutputDataset(train_x, train_y)\n",
    "test_fusion_set = InputOutputDataset(test_x, test_y)\n",
    "\n",
    "train_fusion_loader = DataLoader(train_fusion_set, batch_size=32)\n",
    "test_fusion_loader = DataLoader(test_fusion_set, batch_size=32)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "boost = xgb.XGBClassifier(tree_method='gpu_hist', objective=\"binary:logistic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 144 candidates, totalling 720 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 720 out of 720 | elapsed: 14.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7626681740114671\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'max_depth': [3,6,10],\n",
    "           'learning_rate': [0.0001, 0.005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.3],\n",
    "           'n_estimators': [100, 500, 1000],\n",
    "           'colsample_bytree': [0.3, 0.7]}\n",
    "\n",
    "clf = GridSearchCV(estimator=boost, \n",
    "                   param_grid=param_grid,\n",
    "                   scoring='f1', \n",
    "                   verbose=1)\n",
    "\n",
    "clf.fit(train_x, train_y)\n",
    "\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(MODELS_PATH+'/LateFusion/XGBParams.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(clf.best_params_, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm = SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 85 candidates, totalling 425 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7636471031057497\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 425 out of 425 | elapsed:   12.2s finished\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "    {'C': [0.1, 1, 10, 100, 1000], 'kernel': ['linear']},\n",
    "    {'C': [0.1, 1, 10, 100, 1000], 'gamma': [1, 0.1, 0.01, 0.001], 'kernel': ['rbf']},\n",
    "    {'C': [0.1, 1, 10, 100, 1000], 'gamma': [1, 0.1, 0.01, 0.001], 'degree': [1,2,3], 'kernel': ['poly']}\n",
    "]\n",
    "\n",
    "clf = GridSearchCV(estimator=svm, \n",
    "                   param_grid=param_grid,\n",
    "                   scoring='f1', \n",
    "                   verbose=1)\n",
    "\n",
    "clf.fit(train_x, train_y)\n",
    "\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(MODELS_PATH+'/LateFusion/SVMParams.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(clf.best_params_, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "fusion_model = nn.Sequential(\n",
    "    nn.Linear(in_features=3, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=32),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=32, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=16, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=1),\n",
    ")\n",
    "\n",
    "fusion_model = fusion_model.to(device).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    \"image_model\": None,\n",
    "    \"tabular_model\": None,\n",
    "    \"fusion_model\": fusion_model,\n",
    "}\n",
    "\n",
    "early_stop = {\n",
    "    \"patience\": 20,\n",
    "    \"min_delta\": 0,\n",
    "    \"multip\": 10\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': fusion_model.parameters(), 'lr': 1e-4}],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress:  66%|████████████████████████████▏              | 19/29 [00:00<00:00, 173.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beginning Training: \n",
      "\n",
      "Epoch 1/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 177.62it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 238.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.701096,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 219.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.697569,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 2/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 674.22it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.698096,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.694855,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 3/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 201.20it/s]\n",
      "Batch in Progress:   0%|                                                      | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.694921,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 481.18it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 240.20it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 556.30it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.691984,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 4/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.691564,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.688893,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 5/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 216.18it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 706.75it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.04it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 513.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.687905,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.685461,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 6/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.684097,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 215.91it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 739.56it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.682349,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 7/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.680656,    Accuracy: 48.63%,    Correct Counter: 445/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.679563,    Accuracy: 50.00%,    Correct Counter: 114/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 8/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 199.76it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 572.68it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.65it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 511.88it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.677428,    Accuracy: 51.48%,    Correct Counter: 471/915,    F1 Score: 0.67,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.676729,    Accuracy: 57.02%,    Correct Counter: 130/228,    F1 Score: 0.70,    Precision: 0.54,    Recall:  1.00 \n",
      "\n",
      "Epoch 9/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.674053,    Accuracy: 60.55%,    Correct Counter: 554/915,    F1 Score: 0.71,    Precision: 0.55,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.673685,    Accuracy: 62.28%,    Correct Counter: 142/228,    F1 Score: 0.72,    Precision: 0.57,    Recall:  0.98 \n",
      "\n",
      "Epoch 10/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 208.54it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 1431.81it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 214.66it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 760.75it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.670394,    Accuracy: 66.12%,    Correct Counter: 605/915,    F1 Score: 0.74,    Precision: 0.59,    Recall:  0.99 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.670358,    Accuracy: 63.60%,    Correct Counter: 145/228,    F1 Score: 0.72,    Precision: 0.58,    Recall:  0.96 \n",
      "\n",
      "Epoch 11/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.666383,    Accuracy: 68.63%,    Correct Counter: 628/915,    F1 Score: 0.75,    Precision: 0.61,    Recall:  0.98 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.666690,    Accuracy: 67.11%,    Correct Counter: 153/228,    F1 Score: 0.74,    Precision: 0.61,    Recall:  0.96 \n",
      "\n",
      "Epoch 12/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 237.05it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 600.58it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.74it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 552.93it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.661959,    Accuracy: 70.60%,    Correct Counter: 646/915,    F1 Score: 0.76,    Precision: 0.63,    Recall:  0.97 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.662644,    Accuracy: 65.35%,    Correct Counter: 149/228,    F1 Score: 0.73,    Precision: 0.60,    Recall:  0.93 \n",
      "\n",
      "Epoch 13/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.657052,    Accuracy: 70.93%,    Correct Counter: 649/915,    F1 Score: 0.76,    Precision: 0.63,    Recall:  0.95 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.658201,    Accuracy: 65.35%,    Correct Counter: 149/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.91 \n",
      "\n",
      "Epoch 14/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 237.86it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 729.02it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 215.03it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 733.70it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.651634,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.94 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.653362,    Accuracy: 67.11%,    Correct Counter: 153/228,    F1 Score: 0.73,    Precision: 0.61,    Recall:  0.91 \n",
      "\n",
      "Epoch 15/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.645682,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.76,    Precision: 0.65,    Recall:  0.91 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.648051,    Accuracy: 67.98%,    Correct Counter: 155/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.89 \n",
      "\n",
      "Epoch 16/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 216.76it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 719.76it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.23it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 512.13it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.639216,    Accuracy: 70.60%,    Correct Counter: 646/915,    F1 Score: 0.75,    Precision: 0.64,    Recall:  0.89 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.642330,    Accuracy: 68.86%,    Correct Counter: 157/228,    F1 Score: 0.74,    Precision: 0.63,    Recall:  0.89 \n",
      "\n",
      "Epoch 17/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.632291,    Accuracy: 71.04%,    Correct Counter: 650/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.89 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.636225,    Accuracy: 67.98%,    Correct Counter: 155/228,    F1 Score: 0.73,    Precision: 0.63,    Recall:  0.88 \n",
      "\n",
      "Epoch 18/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 215.39it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 638.12it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.81it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 8/8 [00:00<00:00, 14074.85it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.624921,    Accuracy: 70.93%,    Correct Counter: 649/915,    F1 Score: 0.74,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.629832,    Accuracy: 69.74%,    Correct Counter: 159/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 19/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.617250,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.86 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.623373,    Accuracy: 69.74%,    Correct Counter: 159/228,    F1 Score: 0.74,    Precision: 0.65,    Recall:  0.86 \n",
      "\n",
      "Epoch 20/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 249.54it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 478.77it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.10it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 512.03it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.609451,    Accuracy: 71.37%,    Correct Counter: 653/915,    F1 Score: 0.74,    Precision: 0.66,    Recall:  0.84 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.616984,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 21/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.601738,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.66,    Recall:  0.84 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.610863,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 22/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 220.04it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 580.33it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.95it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 257.51it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.594280,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.83 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.605202,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 23/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.587247,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.74,    Precision: 0.66,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.600104,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 24/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 251.59it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 547.56it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 216.74it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 674.12it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.580806,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.595659,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 25/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.575043,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.591873,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 26/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 220.20it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 585.04it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.44it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 592.98it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.569964,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.588789,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 27/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.565546,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.586327,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 28/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 237.33it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 583.91it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 220.07it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 587.44it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.561758,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.584384,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 29/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.558538,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.582916,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 30/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.82it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 433.08it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.64it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 468.91it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.555759,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.82 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.581863,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 31/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.553372,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.83 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.581111,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 32/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.20it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 583.40it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 256.77it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 512.01it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.551363,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.83 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580580,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 33/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.549675,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.83 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580263,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 34/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 217.98it/s]\n",
      "Batch in Progress: 100%|██████████████████████████████████████████████████████| 8/8 [00:00<?, ?it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.08it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 546.71it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.548234,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.83 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580121,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 35/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.547000,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.84 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580090,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 36/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 216.64it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 707.78it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.25it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 586.56it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.545941,    Accuracy: 72.02%,    Correct Counter: 659/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.84 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580163,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 37/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.545020,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.74,    Precision: 0.67,    Recall:  0.84 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580314,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 38/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 216.83it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 664.83it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.53it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 569.36it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.544212,    Accuracy: 72.02%,    Correct Counter: 659/915,    F1 Score: 0.75,    Precision: 0.67,    Recall:  0.84 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580525,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 39/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.543478,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.75,    Precision: 0.67,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.580783,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 40/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 238.46it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 561.32it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 247.03it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 245.65it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.542834,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.581045,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 41/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.542277,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.581315,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 42/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 219.87it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 585.48it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 248.14it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 475.13it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.541778,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.581597,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 43/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.541326,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.581889,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 44/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.94it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 447.17it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 248.25it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 3233.23it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.540912,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.582196,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.85 \n",
      "\n",
      "Epoch 45/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.540531,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.74,    Precision: 0.66,    Recall:  0.85 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.582496,    Accuracy: 69.74%,    Correct Counter: 159/228,    F1 Score: 0.74,    Precision: 0.65,    Recall:  0.86 \n",
      "\n",
      "Epoch 46/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.27it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 582.16it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 221.79it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 562.88it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.540171,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.86 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.582790,    Accuracy: 70.18%,    Correct Counter: 160/228,    F1 Score: 0.74,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Epoch 47/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.539814,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.86 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.583032,    Accuracy: 69.74%,    Correct Counter: 159/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 48/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.56it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 445.44it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 240.91it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 596.27it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.539451,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.583268,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 49/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.539128,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.583515,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 50/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 210.74it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 1116.32it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.30it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 544.34it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.538830,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.583751,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 51/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.538558,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.583956,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 52/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 249.20it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 465.05it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.65it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 5491.72it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.538321,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.584172,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 53/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.538109,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.75,    Precision: 0.66,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.584390,    Accuracy: 69.30%,    Correct Counter: 158/228,    F1 Score: 0.74,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 54/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.26it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 459.74it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.26it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 562.79it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.537917,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.584607,    Accuracy: 68.86%,    Correct Counter: 157/228,    F1 Score: 0.73,    Precision: 0.64,    Recall:  0.87 \n",
      "\n",
      "Epoch 55/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.537742,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.584820,    Accuracy: 67.98%,    Correct Counter: 155/228,    F1 Score: 0.73,    Precision: 0.63,    Recall:  0.87 \n",
      "\n",
      "Epoch 56/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.28it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 582.43it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.07it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 568.22it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.537582,    Accuracy: 71.37%,    Correct Counter: 653/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.585030,    Accuracy: 67.54%,    Correct Counter: 154/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.87 \n",
      "\n",
      "Epoch 57/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.537432,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.585235,    Accuracy: 67.54%,    Correct Counter: 154/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.87 \n",
      "\n",
      "Epoch 58/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 249.50it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 3067.97it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.70it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 532.24it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.537292,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.585431,    Accuracy: 67.54%,    Correct Counter: 154/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.87 \n",
      "\n",
      "Epoch 59/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.537157,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.585630,    Accuracy: 67.54%,    Correct Counter: 154/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.87 \n",
      "\n",
      "Epoch 60/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 218.64it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 605.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.537027,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.75,    Precision: 0.65,    Recall:  0.87 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.585822,    Accuracy: 67.54%,    Correct Counter: 154/228,    F1 Score: 0.73,    Precision: 0.62,    Recall:  0.87 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs = 1000\n",
    "metrics = train_val(epochs=epochs, models=models, criterion=criterion, optimizer=optimizer, train_loader=train_fusion_loader,\n",
    "                    val_loader=test_fusion_loader, device=device, early_stop=early_stop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(fusion_model.state_dict(), MODELS_PATH + \"\\LateFusion\\FusionPath.pth\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VotingFusion (Late Fusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load image and Tabular model\n",
    "image_model = get_densenet201(device=device, freeze=True, with_mlp=True, outputs=1)\n",
    "\n",
    "tabular_model = nn.Sequential(\n",
    "    nn.Linear(in_features=2, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=32),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=32, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=16, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=1),\n",
    ")\n",
    "\n",
    "image_model.load_state_dict(torch.load(FM_PATH))\n",
    "tabular_model.load_state_dict(torch.load(DM_PATH))\n",
    "\n",
    "image_model, tabular_model = image_model.to(device).float(), tabular_model.to(device).float()\n",
    "\n",
    "# load fusion model\n",
    "tabular_path = nn.Sequential(\n",
    "    nn.Linear(in_features=2, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=32),\n",
    ")\n",
    "\n",
    "image_path = get_densenet201(device=device, freeze=True, with_mlp=False, outputs=32)\n",
    "\n",
    "fusion_path = nn.Sequential(\n",
    "    nn.Linear(in_features=64, out_features=128),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(),\n",
    "    nn.Linear(in_features=128, out_features=32),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(),\n",
    "    nn.Linear(in_features=32, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Dropout(),\n",
    "    nn.Linear(in_features=16, out_features=1),\n",
    ")\n",
    "\n",
    "image_path.load_state_dict(torch.load(MODELS_PATH + \"\\FeatureFusion\\FundusPath.pth\"))\n",
    "tabular_path.load_state_dict(torch.load(MODELS_PATH + \"\\FeatureFusion\\DemographicPath.pth\"))\n",
    "fusion_path.load_state_dict(torch.load(MODELS_PATH + \"\\FeatureFusion\\FusionPath.pth\"))\n",
    "\n",
    "tabular_path, image_path, fusion_path = tabular_path.to(device).float(), image_path.to(device).float(),\\\n",
    "                                                fusion_path.to(device).float()\n",
    "\n",
    "fusion_model = dict()\n",
    "fusion_model[\"image_path\"], fusion_model[\"tabular_path\"], fusion_model[\"fusion_path\"] = image_path, \\\n",
    "                                                                                    tabular_path, fusion_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = {\n",
    "    \"image_model\": image_model,\n",
    "    \"tabular_model\": tabular_model,\n",
    "    \"fusion_model\": fusion_model,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y = build_tabular_dataset(models, train_dataset, method=\"voting_features\")\n",
    "test_x, test_y = build_tabular_dataset(models, test_dataset, method=\"voting_features\")\n",
    "\n",
    "train_fusion_set = InputOutputDataset(train_x, train_y)\n",
    "test_fusion_set = InputOutputDataset(test_x, test_y)\n",
    "\n",
    "train_fusion_loader = DataLoader(train_fusion_set, batch_size=32)\n",
    "test_fusion_loader = DataLoader(test_fusion_set, batch_size=32)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifier using all scores"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "boost = xgb.XGBClassifier(tree_method='gpu_hist', objective=\"binary:logistic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 144 candidates, totalling 720 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "[Parallel(n_jobs=1)]: Done 720 out of 720 | elapsed: 16.2min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7608472256824402\n"
     ]
    }
   ],
   "source": [
    "param_grid = {'max_depth': [3,6,10],\n",
    "           'learning_rate': [0.0001, 0.005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.3],\n",
    "           'n_estimators': [100, 500, 1000],\n",
    "           'colsample_bytree': [0.3, 0.7]}\n",
    "\n",
    "clf = GridSearchCV(estimator=boost, \n",
    "                   param_grid=param_grid,\n",
    "                   scoring='f1', \n",
    "                   verbose=1)\n",
    "\n",
    "clf.fit(train_x, train_y)\n",
    "\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(MODELS_PATH+'/VotingFusion/XGBParams.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(clf.best_params_, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm = SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 85 candidates, totalling 425 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7655842557700883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 425 out of 425 | elapsed:  1.5min finished\n"
     ]
    }
   ],
   "source": [
    "param_grid = [\n",
    "    {'C': [0.1, 1, 10, 100, 1000], 'kernel': ['linear']},\n",
    "    {'C': [0.1, 1, 10, 100, 1000], 'gamma': [1, 0.1, 0.01, 0.001], 'kernel': ['rbf']},\n",
    "    {'C': [0.1, 1, 10, 100, 1000], 'gamma': [1, 0.1, 0.01, 0.001], 'degree': [1,2,3], 'kernel': ['poly']}\n",
    "]\n",
    "\n",
    "clf = GridSearchCV(estimator=svm, \n",
    "                   param_grid=param_grid,\n",
    "                   scoring='f1', \n",
    "                   verbose=1)\n",
    "\n",
    "clf.fit(train_x, train_y)\n",
    "\n",
    "print(clf.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(MODELS_PATH+ '/VotingFusion/SVMParams.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(clf.best_params_, f, ensure_ascii=False, indent=4)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "fusion_model = nn.Sequential(\n",
    "    nn.Linear(in_features=3, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=32),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=32, out_features=16),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=16, out_features=8),\n",
    "    nn.ReLU(inplace=True),\n",
    "    nn.Linear(in_features=8, out_features=1),\n",
    ")\n",
    "\n",
    "fusion_model = fusion_model.to(device).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = {\n",
    "    \"image_model\": None,\n",
    "    \"tabular_model\": None,\n",
    "    \"fusion_model\": fusion_model,\n",
    "}\n",
    "\n",
    "early_stop = {\n",
    "    \"patience\": 20,\n",
    "    \"min_delta\": 0,\n",
    "    \"multip\": 10\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': fusion_model.parameters(), 'lr': 1e-4}],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress:  62%|██████████████████████████▋                | 18/29 [00:00<00:00, 167.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Beginning Training: \n",
      "\n",
      "Epoch 1/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 182.92it/s]\n",
      "Batch in Progress:   0%|                                                      | 0/8 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.700393,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 217.93it/s]\n",
      "Batch in Progress:  66%|████████████████████████████▏              | 19/29 [00:00<00:00, 168.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.697809,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 2/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 170.25it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 272.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.699081,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 223.12it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 910.37it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.696696,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 3/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.697659,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.695481,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 4/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 236.43it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 544.14it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 251.32it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 510.77it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.696066,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.694076,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 5/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.694224,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.692408,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 6/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 222.09it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 309.85it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 219.98it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 267.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.692054,    Accuracy: 48.52%,    Correct Counter: 444/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.690467,    Accuracy: 49.56%,    Correct Counter: 113/228,    F1 Score: 0.66,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 7/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.689546,    Accuracy: 48.63%,    Correct Counter: 445/915,    F1 Score: 0.65,    Precision: 0.49,    Recall:  1.00 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 211.97it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 637.94it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Set:\n",
      "    Average Loss:  0.688255,    Accuracy: 50.44%,    Correct Counter: 115/228,    F1 Score: 0.67,    Precision: 0.50,    Recall:  1.00 \n",
      "\n",
      "Epoch 8/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.686701,    Accuracy: 49.73%,    Correct Counter: 455/915,    F1 Score: 0.66,    Precision: 0.49,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.685775,    Accuracy: 53.07%,    Correct Counter: 121/228,    F1 Score: 0.68,    Precision: 0.51,    Recall:  0.99 \n",
      "\n",
      "Epoch 9/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 215.75it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 728.48it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.37it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 519.88it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.683469,    Accuracy: 53.88%,    Correct Counter: 493/915,    F1 Score: 0.68,    Precision: 0.51,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.682997,    Accuracy: 57.89%,    Correct Counter: 132/228,    F1 Score: 0.70,    Precision: 0.54,    Recall:  0.99 \n",
      "\n",
      "Epoch 10/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.679803,    Accuracy: 60.00%,    Correct Counter: 549/915,    F1 Score: 0.71,    Precision: 0.55,    Recall:  1.00 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.679876,    Accuracy: 60.96%,    Correct Counter: 139/228,    F1 Score: 0.71,    Precision: 0.56,    Recall:  0.98 \n",
      "\n",
      "Epoch 11/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 239.44it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 585.87it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.16it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 581.76it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.675542,    Accuracy: 62.95%,    Correct Counter: 576/915,    F1 Score: 0.72,    Precision: 0.57,    Recall:  0.99 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.676123,    Accuracy: 62.28%,    Correct Counter: 142/228,    F1 Score: 0.72,    Precision: 0.57,    Recall:  0.98 \n",
      "\n",
      "Epoch 12/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.670571,    Accuracy: 65.36%,    Correct Counter: 598/915,    F1 Score: 0.73,    Precision: 0.58,    Recall:  0.99 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.672079,    Accuracy: 62.72%,    Correct Counter: 143/228,    F1 Score: 0.72,    Precision: 0.57,    Recall:  0.96 \n",
      "\n",
      "Epoch 13/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.91it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 497.07it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.69it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 497.96it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.665444,    Accuracy: 67.87%,    Correct Counter: 621/915,    F1 Score: 0.75,    Precision: 0.60,    Recall:  0.99 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.668159,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.73,    Precision: 0.59,    Recall:  0.96 \n",
      "\n",
      "Epoch 14/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.660233,    Accuracy: 68.74%,    Correct Counter: 629/915,    F1 Score: 0.75,    Precision: 0.61,    Recall:  0.98 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.664037,    Accuracy: 66.23%,    Correct Counter: 151/228,    F1 Score: 0.74,    Precision: 0.60,    Recall:  0.96 \n",
      "\n",
      "Epoch 15/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 240.58it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 588.90it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.49it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 547.37it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.654612,    Accuracy: 69.51%,    Correct Counter: 636/915,    F1 Score: 0.76,    Precision: 0.62,    Recall:  0.97 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.659386,    Accuracy: 66.67%,    Correct Counter: 152/228,    F1 Score: 0.74,    Precision: 0.60,    Recall:  0.96 \n",
      "\n",
      "Epoch 16/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.648061,    Accuracy: 69.73%,    Correct Counter: 638/915,    F1 Score: 0.76,    Precision: 0.62,    Recall:  0.97 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.654118,    Accuracy: 66.23%,    Correct Counter: 151/228,    F1 Score: 0.74,    Precision: 0.60,    Recall:  0.95 \n",
      "\n",
      "Epoch 17/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.85it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 480.81it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.47it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 501.07it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.640652,    Accuracy: 69.84%,    Correct Counter: 639/915,    F1 Score: 0.75,    Precision: 0.62,    Recall:  0.96 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.648424,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.92 \n",
      "\n",
      "Epoch 18/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.632734,    Accuracy: 70.05%,    Correct Counter: 641/915,    F1 Score: 0.76,    Precision: 0.63,    Recall:  0.95 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.642637,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.91 \n",
      "\n",
      "Epoch 19/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 236.29it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 649.91it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 240.34it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 563.22it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.624566,    Accuracy: 70.49%,    Correct Counter: 645/915,    F1 Score: 0.76,    Precision: 0.63,    Recall:  0.95 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.637012,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.91 \n",
      "\n",
      "Epoch 20/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.616307,    Accuracy: 70.49%,    Correct Counter: 645/915,    F1 Score: 0.76,    Precision: 0.63,    Recall:  0.95 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.631651,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.91 \n",
      "\n",
      "Epoch 21/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.71it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 528.09it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.93it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 6214.93it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.608144,    Accuracy: 70.71%,    Correct Counter: 647/915,    F1 Score: 0.76,    Precision: 0.63,    Recall:  0.94 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.626690,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.91 \n",
      "\n",
      "Epoch 22/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.600243,    Accuracy: 71.04%,    Correct Counter: 650/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.94 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.622240,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.90 \n",
      "\n",
      "Epoch 23/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 249.09it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 474.76it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.11it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 3260.24it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.592758,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.94 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.618382,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.90 \n",
      "\n",
      "Epoch 24/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.585807,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.94 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.615154,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.90 \n",
      "\n",
      "Epoch 25/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.90it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 567.92it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 252.29it/s]\n",
      "Batch in Progress: 100%|██████████████████████████████████████████████████████| 8/8 [00:00<?, ?it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.579483,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.94 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.612559,    Accuracy: 64.47%,    Correct Counter: 147/228,    F1 Score: 0.72,    Precision: 0.59,    Recall:  0.90 \n",
      "\n",
      "Epoch 26/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.573831,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.610543,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 27/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 236.25it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 861.30it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.55it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 423.03it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.568857,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.609065,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 28/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.564524,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.608052,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 29/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 240.71it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 609.07it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 256.23it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 564.51it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.560777,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.607445,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 30/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.557543,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.607163,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 31/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 247.76it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 502.01it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.34it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 492.43it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.554755,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.607141,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 32/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.552365,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.607292,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 33/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 219.28it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 615.21it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 250.50it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 512.13it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.550326,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.607580,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 34/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.548576,    Accuracy: 71.15%,    Correct Counter: 651/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.607964,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 35/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 232.64it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 1133.60it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.88it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 547.73it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.547072,    Accuracy: 71.26%,    Correct Counter: 652/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.608425,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 36/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.545783,    Accuracy: 71.37%,    Correct Counter: 653/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.608929,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 37/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 246.16it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 516.27it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 218.29it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 641.72it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.544676,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.609480,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 38/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.543740,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.610036,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 39/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.69it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 5601.74it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.31it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 515.85it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.542969,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.610617,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 40/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.542335,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.611167,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 41/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 240.36it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 579.76it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.35it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 2737.57it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.541820,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.611672,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 42/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.541399,    Accuracy: 71.48%,    Correct Counter: 654/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.612139,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 43/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.05it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 574.74it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.66it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 566.57it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.541042,    Accuracy: 71.58%,    Correct Counter: 655/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.612550,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 44/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.540721,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.612917,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 45/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.58it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 557.23it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 218.74it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 642.68it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.540446,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.613236,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 46/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.540191,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.613490,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 47/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.22it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 610.21it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.07it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 468.53it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.539957,    Accuracy: 71.69%,    Correct Counter: 656/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.613697,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 48/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.539735,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.613866,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 49/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.18it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 610.71it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.61it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 467.40it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.539526,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614009,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 50/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.539328,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614121,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 51/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 249.45it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 3079.52it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 248.30it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 473.73it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.539137,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614208,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 52/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.538954,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614271,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 53/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.41it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 5513.38it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 243.65it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 566.20it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.538779,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614304,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 54/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.538613,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614317,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 55/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 244.02it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 5832.51it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 245.71it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 513.07it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.538452,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614307,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 56/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.538298,    Accuracy: 71.80%,    Correct Counter: 657/915,    F1 Score: 0.76,    Precision: 0.64,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614286,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 57/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 241.93it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 607.32it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 216.66it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 654.86it/s]\n",
      "Batch in Progress:   0%|                                                     | 0/29 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.538146,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.76,    Precision: 0.65,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614248,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 58/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.538002,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.76,    Precision: 0.65,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614200,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 59/1000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 242.10it/s]\n",
      "Batch in Progress: 100%|████████████████████████████████████████████| 8/8 [00:00<00:00, 5589.61it/s]\n",
      "Batch in Progress: 100%|███████████████████████████████████████████| 29/29 [00:00<00:00, 248.11it/s]\n",
      "Batch in Progress: 100%|█████████████████████████████████████████████| 8/8 [00:00<00:00, 475.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Set:\n",
      "    Average Loss:  0.537861,    Accuracy: 71.91%,    Correct Counter: 658/915,    F1 Score: 0.76,    Precision: 0.65,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614149,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n",
      "Epoch 60/1000\n",
      "Train Set:\n",
      "    Average Loss:  0.537722,    Accuracy: 72.13%,    Correct Counter: 660/915,    F1 Score: 0.76,    Precision: 0.65,    Recall:  0.93 \n",
      "\n",
      "Validation Set:\n",
      "    Average Loss:  0.614096,    Accuracy: 64.91%,    Correct Counter: 148/228,    F1 Score: 0.72,    Precision: 0.60,    Recall:  0.90 \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "epochs = 1000\n",
    "metrics = train_val(epochs=epochs, models=model, criterion=criterion, optimizer=optimizer, train_loader=train_fusion_loader,\n",
    "                    val_loader=test_fusion_loader, device=device, early_stop=early_stop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(fusion_model.state_dict(), MODELS_PATH + \"\\VotingFusion\\FusionPath.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run C:/Users/MohammedSB/Desktop/projects/Hypertension/Results.ipynb"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}