casme2_me_main_dev.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set `autoreload` to execute the change in `.py` files.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle\n",
    "import natsort\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "from __utils__ import image_processing\n",
    "from __utils__ import label_processing\n",
    "from __utils__ import labeling\n",
    "from __utils__ import loso_preparing\n",
    "from __utils__ import functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preds_path: D:\\Databases\\CAS(ME)^2\\preds\\me_sl_swin_t_batch_size_48_epochs_25_pseudo_labeling_128_3.pkl\n"
     ]
    }
   ],
   "source": [
    "dataset_dir = \"D:/Databases/CAS(ME)^2\"\n",
    "# dataset_dir = \"F:/HEH/Databases/CAS(ME)^2\"\n",
    "# dataset_dir = \"/data/disk1/heh/databases/CAS(ME)^2\"\n",
    "\n",
    "images_loading = False\n",
    "image_size = 128\n",
    "load_cropped_images = False\n",
    "# expression_type = \"mae\"  # macro-expression spotting\n",
    "expression_type = \"me\"  # micro-expression spotting\n",
    "debug_preds = True\n",
    "labeling_function = \"pseudo_labeling\"\n",
    "# labeling_function = \"original_labeling\"\n",
    "model_names = {\n",
    "    0: \"SOFTNet\",\n",
    "    1: \"SOFTNetCBAM\",\n",
    "    2: \"ViT-B\",\n",
    "    3: \"SL-ViT-B\",\n",
    "    4: \"Swin-T\",\n",
    "    5: \"Swin-S\",\n",
    "    6: \"L-Swin-T\",\n",
    "    7: \"S-Swin-T\",\n",
    "    8: \"SL-Swin-T\",\n",
    "    9: \"SL-Swin-S\",\n",
    "}\n",
    "model_name = model_names[8]\n",
    "batch_size = 48\n",
    "epochs = 25\n",
    "save_preds = False\n",
    "preds_stem = (\n",
    "    f\"{expression_type}_\"\n",
    "    + model_name.lower().replace(\"-\", \"_\")\n",
    "    + f\"_batch_size_{batch_size}\"\n",
    "    + f\"_epochs_{epochs}\"\n",
    "    + f\"_{labeling_function}\"\n",
    "    + f\"_{image_size}\"\n",
    "    + \"_3\"\n",
    ")\n",
    "preds_path = Path(dataset_dir, \"preds\", preds_stem).with_suffix(\".pkl\")\n",
    "print(f\"preds_path: {preds_path}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Images\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When debug the image processing, the videos_images is from cropped_rawpic, whereas the other variables are from rawpic.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subject:  s15\n",
      "subject:  s16\n",
      "subject:  s19\n",
      "subject:  s20\n",
      "subject:  s21\n",
      "subject:  s22\n",
      "subject:  s23\n",
      "subject:  s24\n",
      "subject:  s25\n",
      "subject:  s26\n",
      "subject:  s27\n",
      "subject:  s29\n",
      "subject:  s30\n",
      "subject:  s31\n",
      "subject:  s32\n",
      "subject:  s33\n",
      "subject:  s34\n",
      "subject:  s35\n",
      "subject:  s36\n",
      "subject:  s37\n",
      "subject:  s38\n",
      "subject:  s40\n"
     ]
    }
   ],
   "source": [
    "videos_images, subjects, subjects_videos_code = image_processing.load_images(\n",
    "    dataset_dir,\n",
    "    images_loading=images_loading,\n",
    "    image_size=image_size,\n",
    "    load_cropped_images=load_cropped_images,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subjects: ['s15', 's16', 's19', 's20', 's21', 's22', 's23', 's24', 's25', 's26', 's27', 's29', 's30', 's31', 's32', 's33', 's34', 's35', 's36', 's37', 's38', 's40']\n",
      "subjects_videos_code: [['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0508'], ['0101', '0102', '0401', '0402', '0502', '0505', '0507'], ['0102', '0402', '0505', '0507', '0502'], ['0502'], ['0101', '0401'], ['0101', '0102', '0402', '0503', '0508'], ['0102', '0402', '0503', '0507'], ['0101', '0401', '0402', '0502', '0507'], ['0101', '0102', '0502', '0508'], ['0101', '0102', '0401', '0503'], ['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0507', '0508'], ['0502'], ['0101', '0102', '0401', '0502', '0503', '0505', '0507'], ['0101', '0401', '0402', '0502', '0503', '0505', '0507'], ['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0507', '0508'], ['0102', '0402'], ['0401', '0402', '0503'], ['0102'], ['0401', '0505'], ['0101', '0402', '0502', '0505', '0507', '0508'], ['0502', '0507'], ['0401', '0502', '0503']]\n"
     ]
    }
   ],
   "source": [
    "print(\"subjects:\", subjects)\n",
    "print(\"subjects_videos_code:\", subjects_videos_code)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Excel \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant</th>\n",
       "      <th>video_name_&amp;_expression_number</th>\n",
       "      <th>onset</th>\n",
       "      <th>apex</th>\n",
       "      <th>offset</th>\n",
       "      <th>AUs</th>\n",
       "      <th>extimated_emotion</th>\n",
       "      <th>expression_type</th>\n",
       "      <th>self-reported_emotion</th>\n",
       "      <th>video_name</th>\n",
       "      <th>video_code</th>\n",
       "      <th>subject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>anger1_1</td>\n",
       "      <td>557</td>\n",
       "      <td>572</td>\n",
       "      <td>608</td>\n",
       "      <td>4+10+14+15</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger1</td>\n",
       "      <td>0401</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>anger1_2</td>\n",
       "      <td>2854</td>\n",
       "      <td>2862</td>\n",
       "      <td>2871</td>\n",
       "      <td>38</td>\n",
       "      <td>others</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>sadness</td>\n",
       "      <td>anger1</td>\n",
       "      <td>0401</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>anger2_1</td>\n",
       "      <td>2155</td>\n",
       "      <td>2163</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger2</td>\n",
       "      <td>0402</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>anger2_2</td>\n",
       "      <td>3363</td>\n",
       "      <td>3371</td>\n",
       "      <td>3383</td>\n",
       "      <td>4+7+14</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger2</td>\n",
       "      <td>0402</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>anger2_3</td>\n",
       "      <td>3380</td>\n",
       "      <td>3386</td>\n",
       "      <td>3407</td>\n",
       "      <td>4+14+38</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger2</td>\n",
       "      <td>0402</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   participant video_name_&_expression_number  onset  apex  offset   \n",
       "0            1                       anger1_1    557   572     608  \\\n",
       "1            1                       anger1_2   2854  2862    2871   \n",
       "2            1                       anger2_1   2155  2163       0   \n",
       "3            1                       anger2_2   3363  3371    3383   \n",
       "4            1                       anger2_3   3380  3386    3407   \n",
       "\n",
       "          AUs extimated_emotion   expression_type self-reported_emotion   \n",
       "0  4+10+14+15          negative  macro-expression                 anger  \\\n",
       "1          38            others  macro-expression               sadness   \n",
       "2         NaN          negative  macro-expression                 anger   \n",
       "3      4+7+14          negative  macro-expression                 anger   \n",
       "4     4+14+38          negative  macro-expression                 anger   \n",
       "\n",
       "  video_name video_code subject  \n",
       "0     anger1       0401     s15  \n",
       "1     anger1       0401     s15  \n",
       "2     anger2       0402     s15  \n",
       "3     anger2       0402     s15  \n",
       "4     anger2       0402     s15  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Excel_data = label_processing.load_excel(dataset_dir)\n",
    "Excel_data.head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Ground Truth Labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "required_videos_index:  [1, 4, 8, 9, 12, 13, 14, 16, 28, 33, 36, 37, 38, 45, 46, 47, 49, 50, 52, 54, 55, 57, 62, 64, 67, 71, 73, 74, 77, 83, 87, 91, 93]\n",
      "len(clean_videos_images) = 33\n"
     ]
    }
   ],
   "source": [
    "(\n",
    "    clean_videos_images,\n",
    "    clean_subjects_videos_code,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    ") = label_processing.load_ground_truth_labels(\n",
    "    dataset_dir,\n",
    "    expression_type,\n",
    "    videos_images,\n",
    "    subjects_videos_code,\n",
    "    subjects,\n",
    "    Excel_data,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(clean_subjects):  14\n",
      "clean_subjects:  ['s15' 's16' 's19' 's23' 's24' 's25' 's27' 's29' 's30' 's31' 's32' 's35'\n",
      " 's37' 's38']\n",
      "len(clean_subjects_videos_code):  14\n",
      "clean_subjects_videos_codes:  [['0102', '0502'], ['0101', '0102', '0502', '0505', '0507'], ['0402'], ['0102'], ['0401', '0507'], ['0101', '0102'], ['0101', '0102', '0401', '0502', '0503', '0507'], ['0502'], ['0101', '0401'], ['0101', '0402', '0505'], ['0401', '0502', '0503', '0508'], ['0102'], ['0402', '0508'], ['0507']]\n",
      "len(clean_subjects_videos_ground_truth_labels):  14\n",
      "clean_subjects_videos_ground_truth_labels:  [[[[698, 706]], [[137, 147]]], [[[551, 564]], [[269, 277]], [[322, 333]], [[395, 406], [1694, 1709], [1879, 1894]], [[1957, 1967], [2284, 2294]]], [[[1926, 1941]]], [[[330, 345], [525, 539], [726, 739]]], [[[607, 620], [962, 976], [1889, 1901], [2180, 2192], [3440, 3452]], [[1835, 1847], [1950, 1964], [3232, 3247]]], [[[112, 126]], [[995, 1007], [1007, 1016], [1017, 1033]]], [[[873, 887]], [[33, 47], [308, 316], [373, 387]], [[351, 364], [368, 381], [1134, 1146], [1973, 1985]], [[612, 627]], [[418, 431]], [[875, 889]]], [[[139, 151]]], [[[1454, 1465]], [[925, 940]]], [[[1420, 1432]], [[1688, 1701], [2189, 2203], [2376, 2388], [3802, 3814]], [[1045, 1058]]], [[[267, 277]], [[310, 323], [1170, 1183]], [[257, 271], [1030, 1043]], [[285, 300]]], [[[99, 112], [362, 370]]], [[[3501, 3513]], [[417, 429]]], [[[2231, 2246]]]]\n"
     ]
    }
   ],
   "source": [
    "print(\"len(clean_subjects): \", len(clean_subjects))\n",
    "print(\"clean_subjects: \", clean_subjects)\n",
    "print(\"len(clean_subjects_videos_code): \", len(clean_subjects_videos_code))\n",
    "print(\"clean_subjects_videos_codes: \", clean_subjects_videos_code)\n",
    "print(\n",
    "    \"len(clean_subjects_videos_ground_truth_labels): \",\n",
    "    len(clean_subjects_videos_ground_truth_labels),\n",
    ")\n",
    "print(\n",
    "    \"clean_subjects_videos_ground_truth_labels: \",\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate `k`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k (Half of average length of expression) =  6\n"
     ]
    }
   ],
   "source": [
    "k = label_processing.calculate_k(clean_subjects_videos_ground_truth_labels)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Labeling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total frames: 80463\n"
     ]
    }
   ],
   "source": [
    "if labeling_function == \"pseudo_labeling\":\n",
    "    labels = labeling.get_pseudo_labels(\n",
    "        clean_videos_images, clean_subjects_videos_ground_truth_labels, k\n",
    "    )\n",
    "elif labeling_function == \"original_labeling\":\n",
    "    labels = labeling.get_original_labels(\n",
    "        clean_videos_images, clean_subjects_videos_ground_truth_labels, k\n",
    "    )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare for LOSO\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frame Index for each subject:-\n",
      "\n",
      "subject s15 ( group = 0): 0 -> 3336\n",
      "subject s15 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  2\n",
      "\n",
      "subject s16 ( group = 1): 3336 -> 14273\n",
      "subject s16 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  7\n",
      "\n",
      "subject s19 ( group = 2): 14273 -> 18638\n",
      "subject s19 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  8\n",
      "\n",
      "subject s23 ( group = 3): 18638 -> 19703\n",
      "subject s23 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  9\n",
      "\n",
      "subject s24 ( group = 4): 19703 -> 26673\n",
      "subject s24 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  11\n",
      "\n",
      "subject s25 ( group = 5): 26673 -> 29790\n",
      "subject s25 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  13\n",
      "\n",
      "subject s27 ( group = 6): 29790 -> 44942\n",
      "subject s27 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  19\n",
      "\n",
      "subject s29 ( group = 7): 44942 -> 47211\n",
      "subject s29 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  20\n",
      "\n",
      "subject s30 ( group = 8): 47211 -> 52966\n",
      "subject s30 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  22\n",
      "\n",
      "subject s31 ( group = 9): 52966 -> 61673\n",
      "subject s31 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  25\n",
      "\n",
      "subject s32 ( group = 10): 61673 -> 71111\n",
      "subject s32 has 4 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  29\n",
      "\n",
      "subject s35 ( group = 11): 71111 -> 72186\n",
      "subject s35 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  30\n",
      "\n",
      "subject s37 ( group = 12): 72186 -> 77205\n",
      "subject s37 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  32\n",
      "\n",
      "subject s38 ( group = 13): 77205 -> 80463\n",
      "subject s38 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  33\n"
     ]
    }
   ],
   "source": [
    "y, groups = loso_preparing.prepare_for_loso(\n",
    "    labels,\n",
    "    clean_subjects,\n",
    "    clean_videos_images,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    k,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if debug_preds is False:\n",
    "    from __utils__.training_dev import train\n",
    "\n",
    "    preds = train(\n",
    "        dataset_dir=dataset_dir,\n",
    "        clean_subjects=clean_subjects,\n",
    "        image_size=image_size,\n",
    "        y=y,\n",
    "        expression_type=expression_type,\n",
    "        model_name=model_name,\n",
    "        train_or_not=True,\n",
    "        batch_size=batch_size,\n",
    "        epochs=epochs,\n",
    "        preds_path=preds_path,\n",
    "    )\n",
    "else:\n",
    "    with open(preds_path, \"rb\") as pkl_file:\n",
    "        preds = _pickle.load(pkl_file)\n",
    "        pkl_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "if save_preds is True:\n",
    "    with open(preds_path, \"wb\") as pkl_file:\n",
    "        _pickle.dump(preds, pkl_file)\n",
    "        pkl_file.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spotting and Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1/14 is in process.\n",
      "0 video(s) have been processed.\n",
      "The current video be processed: subject s15, video 0102\n",
      "The current video be processed: subject s15, video 0502\n",
      "\n",
      "True Positive: 0, False Posive: 7, False Negative: 2\n",
      "Precision = 2.220446049250313e-16, Recall =2.220446049250313e-16, F1-Score = 4.440892098500626e-16\n",
      "Split 1/14 is processed.\n",
      "\n",
      "Split 2/14 is in process.\n",
      "2 video(s) have been processed.\n",
      "The current video be processed: subject s16, video 0101\n",
      "The current video be processed: subject s16, video 0102\n",
      "The current video be processed: subject s16, video 0502\n",
      "The current video be processed: subject s16, video 0505\n",
      "The current video be processed: subject s16, video 0507\n",
      "\n",
      "True Positive: 0, False Posive: 29, False Negative: 10\n",
      "Precision = 2.220446049250313e-16, Recall =2.220446049250313e-16, F1-Score = 4.440892098500626e-16\n",
      "Split 2/14 is processed.\n",
      "\n",
      "Split 3/14 is in process.\n",
      "7 video(s) have been processed.\n",
      "The current video be processed: subject s19, video 0402\n",
      "\n",
      "True Positive: 0, False Posive: 34, False Negative: 11\n",
      "Precision = 2.220446049250313e-16, Recall =2.220446049250313e-16, F1-Score = 4.440892098500626e-16\n",
      "Split 3/14 is processed.\n",
      "\n",
      "Split 4/14 is in process.\n",
      "8 video(s) have been processed.\n",
      "The current video be processed: subject s23, video 0102\n",
      "\n",
      "True Positive: 0, False Posive: 54, False Negative: 14\n",
      "Precision = 2.220446049250313e-16, Recall =2.220446049250313e-16, F1-Score = 4.440892098500626e-16\n",
      "Split 4/14 is processed.\n",
      "\n",
      "Split 5/14 is in process.\n",
      "9 video(s) have been processed.\n",
      "The current video be processed: subject s24, video 0401\n",
      "The current video be processed: subject s24, video 0507\n",
      "\n",
      "True Positive: 0, False Posive: 61, False Negative: 22\n",
      "Precision = 2.220446049250313e-16, Recall =2.220446049250313e-16, F1-Score = 4.440892098500626e-16\n",
      "Split 5/14 is processed.\n",
      "\n",
      "Split 6/14 is in process.\n",
      "11 video(s) have been processed.\n",
      "The current video be processed: subject s25, video 0101\n",
      "The current video be processed: subject s25, video 0102\n",
      "\n",
      "True Positive: 4, False Posive: 75, False Negative: 22\n",
      "Precision = 0.050632911392405285, Recall =0.15384615384615408, F1-Score = 0.0761904761904767\n",
      "Split 6/14 is processed.\n",
      "\n",
      "Split 7/14 is in process.\n",
      "13 video(s) have been processed.\n",
      "The current video be processed: subject s27, video 0101\n",
      "The current video be processed: subject s27, video 0102\n",
      "The current video be processed: subject s27, video 0401\n",
      "The current video be processed: subject s27, video 0502\n",
      "The current video be processed: subject s27, video 0503\n",
      "The current video be processed: subject s27, video 0507\n",
      "\n",
      "True Positive: 8, False Posive: 100, False Negative: 29\n",
      "Precision = 0.07407407407407429, Recall =0.21621621621621645, F1-Score = 0.1103448275862074\n",
      "Split 7/14 is processed.\n",
      "\n",
      "Split 8/14 is in process.\n",
      "19 video(s) have been processed.\n",
      "The current video be processed: subject s29, video 0502\n",
      "\n",
      "True Positive: 8, False Posive: 109, False Negative: 30\n",
      "Precision = 0.0683760683760686, Recall =0.2105263157894739, F1-Score = 0.10322580645161343\n",
      "Split 8/14 is processed.\n",
      "\n",
      "Split 9/14 is in process.\n",
      "20 video(s) have been processed.\n",
      "The current video be processed: subject s30, video 0101\n",
      "The current video be processed: subject s30, video 0401\n",
      "\n",
      "True Positive: 9, False Posive: 146, False Negative: 31\n",
      "Precision = 0.05806451612903248, Recall =0.22500000000000023, F1-Score = 0.09230769230769283\n",
      "Split 9/14 is processed.\n",
      "\n",
      "Split 10/14 is in process.\n",
      "22 video(s) have been processed.\n",
      "The current video be processed: subject s31, video 0101\n",
      "The current video be processed: subject s31, video 0402\n",
      "The current video be processed: subject s31, video 0505\n",
      "\n",
      "True Positive: 10, False Posive: 155, False Negative: 36\n",
      "Precision = 0.06060606060606083, Recall =0.2173913043478263, F1-Score = 0.09478672985782041\n",
      "Split 10/14 is processed.\n",
      "\n",
      "Split 11/14 is in process.\n",
      "25 video(s) have been processed.\n",
      "The current video be processed: subject s32, video 0401\n",
      "The current video be processed: subject s32, video 0502\n",
      "The current video be processed: subject s32, video 0503\n",
      "The current video be processed: subject s32, video 0508\n",
      "\n",
      "True Positive: 11, False Posive: 189, False Negative: 41\n",
      "Precision = 0.05500000000000022, Recall =0.21153846153846176, F1-Score = 0.08730158730158782\n",
      "Split 11/14 is processed.\n",
      "\n",
      "Split 12/14 is in process.\n",
      "29 video(s) have been processed.\n",
      "The current video be processed: subject s35, video 0102\n",
      "\n",
      "True Positive: 11, False Posive: 190, False Negative: 43\n",
      "Precision = 0.0547263681592042, Recall =0.20370370370370391, F1-Score = 0.08627450980392208\n",
      "Split 12/14 is processed.\n",
      "\n",
      "Split 13/14 is in process.\n",
      "30 video(s) have been processed.\n",
      "The current video be processed: subject s37, video 0402\n",
      "The current video be processed: subject s37, video 0508\n",
      "\n",
      "True Positive: 12, False Posive: 198, False Negative: 44\n",
      "Precision = 0.05714285714285736, Recall =0.2142857142857145, F1-Score = 0.09022556390977494\n",
      "Split 13/14 is processed.\n",
      "\n",
      "Split 14/14 is in process.\n",
      "32 video(s) have been processed.\n",
      "The current video be processed: subject s38, video 0507\n",
      "\n",
      "True Positive: 12, False Posive: 204, False Negative: 45\n",
      "Precision = 0.055555555555555775, Recall =0.2105263157894739, F1-Score = 0.08791208791208842\n",
      "Split 14/14 is processed.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metric_fn, result_dict = functions.spot_and_evaluate(\n",
    "    preds,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    clean_videos_images,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_code,\n",
    "    k,\n",
    "    p=0.60,\n",
    "    show_plot_or_not=False,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Positive: 12, False Posive: 204, False Negative: 45\n",
      "Final Precision = 0.055555555555555775,\n",
      "Final Recall =0.2105263157894739,\n",
      "Final F1-Score = 0.08791208791208842\n",
      "\n",
      "Highest Precision = 0.07407407407407429,\n",
      "Highest Recall =0.22500000000000023,\n",
      "Highest F1-Score = 0.1103448275862074\n"
     ]
    }
   ],
   "source": [
    "functions.final_evaluate(metric_fn, result_dict)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 256\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value\n",
    "| --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T\n",
    "| epochs | 20 (331 m) | 25 (775 m)\n",
    "| batch_size | 48 | 96\n",
    "| p | 0.60 | 0.60 | \n",
    "| True Positive | 0 | 13 | \n",
    "| False Positive | 0 | 339 | \n",
    "| False Negative | 0 | 44 | \n",
    "| Precision | 0. | 0.00369 | \n",
    "| Recall | 0. | 0.2280 | \n",
    "| F1-Score | 0.1000 | 0.0635 |  \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 128\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value\n",
    "| --- | --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T\n",
    "| epochs | 20 (505 m) | 25 (593 m) | 30 (695 m)\n",
    "| batch_size | 32 | 32 | 32\n",
    "| p | 0.86 | 0.77 | | 0.59\n",
    "| True Positive | 7 | 5 | \n",
    "| False Positive | 51 | 88 | \n",
    "| False Negative | 50 | 52 | \n",
    "| Precision | 0.1207 | 0.0537 | \n",
    "| Recall | 0.1228 | 0.0877 | \n",
    "| F1-Score | 0.1217 | 0.0666 |  \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value\n",
    "| --- | --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T\n",
    "| epochs | 20 (311 m) | 20 (311 m) | 30 (410 m)\n",
    "| batch_size | 48 | 48 | 48\n",
    "| p | 0.57 | 0.59 | 0.55\n",
    "| True Positive | 15 | 15 | 13\n",
    "| False Positive | 244 | 221 | 289\n",
    "| False Negative | 42 | 42 | 44\n",
    "| Precision | 0.0579 | 0.0636 | 0.0430\n",
    "| Recall | 0.2632 | 0.2632 | 0.2280\n",
    "| F1-Score | 0.0949 | 0.1024 | 0.0724\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value | Value\n",
    "| --- | --- | --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T_2 | SL-Swin-T_3\n",
    "| epochs | 25 (325 m) | 25 (325 m) | 25 (325 m) | 25 (325 m)\n",
    "| batch_size | 48 | 48 | 48 | 48\n",
    "| p | 0.57 | 0.59 | 0.60 | 0.60\n",
    "| True Positive | 14 | 14 | 13 | 12\n",
    "| False Positive | 266 | 239 | 226 | 204\n",
    "| False Negative | 43 | 43 | 44 | 45\n",
    "| Precision | 0.0500 | 0.0553 | 0.0544 | 0.0556\n",
    "| Recall | 0.2456 | 0.2456 | 0.2281 | 0.2105\n",
    "| F1-Score | 0.0830 | 0.0903 | 0.0878 | 0.0879\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value | Value | Value\n",
    "| --- | --- | --- | --- | --- | ---\n",
    "| model | ViT-B | SL-ViT-B | Swin-T | L-Swin-T | S-Swin-T\n",
    "| epochs | 25 ( m) | 25 ( m) | 25 ( m) | 25 (568 m) | 25 (364 m)\n",
    "| batch_size | 96 | 96 | 48 | 48 | 48\n",
    "| p | 0. | 0. | 0.57 | 0.51 | 0.49 | \n",
    "| True Positive | 0 | 0 | 13 | 10 | 16 | \n",
    "| False Positive | 0 | 0 | 262 | 339 | 354 | \n",
    "| False Negative | 0 | 0 | 44 | 47 | 41 | \n",
    "| Precision | 0. | 0 | 0.0473 | 0.0287 | 0.0432 | \n",
    "| Recall | 0. | 0 | 0.2281 | 0.1754 | 0.2807 | \n",
    "| F1-Score | 0. | 0 | 0.0783 | 0.0493 | 0.0749 | \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Original Labeling\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value |\n",
    "| --- | --- | --- |\n",
    "| model | Swin-T | SL-Swin-T |\n",
    "| epochs | 25 | 25 |\n",
    "| batch_size | 48 | 48 |\n",
    "| p | 0.58 | 0.57 |\n",
    "| True Positive | 8 | 15 |\n",
    "| False Positive | 237 | 305 |\n",
    "| False Negative | 49 | 42 |\n",
    "| Precision | 0.0327 | 0.0469 |\n",
    "| Recall | 0.1404 | 0.2632 |\n",
    "| F1-Score | 0.0503 | 0.0796 |\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stretched (Deprecated)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value (tf) | Value (tf) | Value | Value\n",
    "| --- | --- | --- | --- | --- | --- | ---\n",
    "| model | 3D-CNN | SOFTNet | SOFTNet | SOFTNet (dev) | SOFTNet | SOFTNet\n",
    "| epochs | | 10 | 20 (39 m)| 20 (39 m) | 35 (54 m) | 20 (28 m)\n",
    "| batch_size | | 48 | 48 | 48 | 48 | 48\n",
    "| learning_rate | | 0.0005 | 0.0005 | 0.0005 | 0.0005 | 0.0005\n",
    "| True Positive | | 20 | 19 | 16 | 20 | 20\n",
    "| False Positive | | 264 | 252 | 238 | 285 | 409\n",
    "| False Negative | | 37 | 38 | 41 | 37 | 37\n",
    "| Precision | | 0.0704 | 0.0701 | 0.0629 | 0.0655 | 0.0466\n",
    "| Recall | | 0.3509 | 0.3333 | 0.2807 | 0.3508 | 0.3508\n",
    "| F1-Score | 0.0714 | 0.1173 | 0.1159 | 0.1028 | 0.1104 | 0.0823\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "without pos_embedding\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value 1 | Value 2 | Value 3 | Value 4 | Value 5 | Value 6 | Value 7\n",
    "| --- | --- | --- | --- | --- | --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T | SL-Swin-T | SL-Swin-T | SL-Swin-T | SL-Swin-T\n",
    "| epochs | 20 (392 m) | 25 (531 m) | 30 (581 m) | 20 (382 m) | 30 (437 m) | 35 (499 m) | 50 (784 m)\n",
    "| batch_size | 32 | 32 | 32 | 48 | 48 | 48 | 48\n",
    "| learning_rate | 0.0005 | 0.0005 | 0.0005 | 0.0005 | 0.0005 | 0.0005 | 0.0005\n",
    "| True Positive | 14 | 17 | 11 | 10 | 11 | 11 | 11\n",
    "| False Positive | 354 | 363 | 360 | 366 | 348 | 357 | 380\n",
    "| False Negative | 43 | 40 | 46 | 47 | 46 | 46 | 46\n",
    "| Precision | 0.0380 | 0.0447 | 0.0296 | 0.0265 | 0.0306 | 0.0298 | 0.0281\n",
    "| Recall | 0.2456 | 0.2982 | 0.1929 | 0.1754 | 0.1929 | 0.1929 | 0.1929\n",
    "| F1-Score | 0.0658 | 0.0778 | 0.0514 | 0.0461 | 0.0528 | 0.0517 | 0.0491\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "with pos_embedding\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value | Value | Value\n",
    "| --- | --- | --- | --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T | SL-Swin-T | SL-Swin-T\n",
    "| epochs | 25 (987 m) | 20 (389 m) | 25 (489 m) | 30 (611 m) | 25 (350 m)\n",
    "| batch_size | 16 | 32 | 32 | 32 | 48\n",
    "| learning_rate | 0.0005 | 0.0005 | 0.0005 | 0.0005 | 0.0005\n",
    "| True Positive | 11 | 12 | 16 | 13 | 11\n",
    "| False Positive | 325 | 342 | 370 | 335 | 371\n",
    "| False Negative | 46 | 45 | 41 | 44 | 46\n",
    "| Precision | 0.0327 | 0.0338 | 0.0414 | 0.0375 | 0.0287\n",
    "| Recall | 0.1929 | 0.2105 | 0.2807 | 0.2280 | 0.1929\n",
    "| F1-Score | 0.0559 | 0.0583 | 0.0722 | 0.0641 | 0.0501\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ablation Study\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " p | TP | FP | FN | Precision | Recall | F1-Score\n",
      "0.01 | 44 | 4705 | 13 | 0.0093 | 0.7719 | 0.0183 |\n",
      "0.02 | 44 | 4482 | 13 | 0.0097 | 0.7719 | 0.0192 |\n",
      "0.03 | 44 | 4287 | 13 | 0.0102 | 0.7719 | 0.0201 |\n",
      "0.04 | 44 | 4071 | 13 | 0.0107 | 0.7719 | 0.0211 |\n",
      "0.05 | 44 | 3889 | 13 | 0.0112 | 0.7719 | 0.0221 |\n",
      "0.06 | 41 | 3697 | 16 | 0.0110 | 0.7193 | 0.0216 |\n",
      "0.07 | 41 | 3513 | 16 | 0.0115 | 0.7193 | 0.0227 |\n",
      "0.08 | 39 | 3320 | 18 | 0.0116 | 0.6842 | 0.0228 |\n",
      "0.09 | 38 | 3147 | 19 | 0.0119 | 0.6667 | 0.0234 |\n",
      "0.10 | 37 | 2970 | 20 | 0.0123 | 0.6491 | 0.0242 |\n",
      "0.11 | 37 | 2793 | 20 | 0.0131 | 0.6491 | 0.0256 |\n",
      "0.12 | 36 | 2632 | 21 | 0.0135 | 0.6316 | 0.0264 |\n",
      "0.13 | 34 | 2505 | 23 | 0.0134 | 0.5965 | 0.0262 |\n",
      "0.14 | 34 | 2370 | 23 | 0.0141 | 0.5965 | 0.0276 |\n",
      "0.15 | 34 | 2230 | 23 | 0.0150 | 0.5965 | 0.0293 |\n",
      "0.16 | 33 | 2109 | 24 | 0.0154 | 0.5789 | 0.0300 |\n",
      "0.17 | 32 | 1987 | 25 | 0.0158 | 0.5614 | 0.0308 |\n",
      "0.18 | 32 | 1873 | 25 | 0.0168 | 0.5614 | 0.0326 |\n",
      "0.19 | 30 | 1763 | 27 | 0.0167 | 0.5263 | 0.0324 |\n",
      "0.20 | 30 | 1674 | 27 | 0.0176 | 0.5263 | 0.0341 |\n",
      "0.21 | 29 | 1586 | 28 | 0.0180 | 0.5088 | 0.0347 |\n",
      "0.22 | 29 | 1516 | 28 | 0.0188 | 0.5088 | 0.0362 |\n",
      "0.23 | 27 | 1431 | 30 | 0.0185 | 0.4737 | 0.0356 |\n",
      "0.24 | 27 | 1351 | 30 | 0.0196 | 0.4737 | 0.0376 |\n",
      "0.25 | 27 | 1291 | 30 | 0.0205 | 0.4737 | 0.0393 |\n",
      "0.26 | 25 | 1229 | 32 | 0.0199 | 0.4386 | 0.0381 |\n",
      "0.27 | 24 | 1168 | 33 | 0.0201 | 0.4211 | 0.0384 |\n",
      "0.28 | 23 | 1127 | 34 | 0.0200 | 0.4035 | 0.0381 |\n",
      "0.29 | 23 | 1072 | 34 | 0.0210 | 0.4035 | 0.0399 |\n",
      "0.30 | 23 | 1018 | 34 | 0.0221 | 0.4035 | 0.0419 |\n",
      "0.31 | 22 | 972 | 35 | 0.0221 | 0.3860 | 0.0419 |\n",
      "0.32 | 21 | 919 | 36 | 0.0223 | 0.3684 | 0.0421 |\n",
      "0.33 | 21 | 878 | 36 | 0.0234 | 0.3684 | 0.0439 |\n",
      "0.34 | 19 | 836 | 38 | 0.0222 | 0.3333 | 0.0417 |\n",
      "0.35 | 19 | 790 | 38 | 0.0235 | 0.3333 | 0.0439 |\n",
      "0.36 | 19 | 754 | 38 | 0.0246 | 0.3333 | 0.0458 |\n",
      "0.37 | 19 | 705 | 38 | 0.0262 | 0.3333 | 0.0487 |\n",
      "0.38 | 18 | 678 | 39 | 0.0259 | 0.3158 | 0.0478 |\n",
      "0.39 | 18 | 643 | 39 | 0.0272 | 0.3158 | 0.0501 |\n",
      "0.40 | 18 | 605 | 39 | 0.0289 | 0.3158 | 0.0529 |\n",
      "0.41 | 17 | 569 | 40 | 0.0290 | 0.2982 | 0.0529 |\n",
      "0.42 | 17 | 534 | 40 | 0.0309 | 0.2982 | 0.0559 |\n",
      "0.43 | 16 | 505 | 41 | 0.0307 | 0.2807 | 0.0554 |\n",
      "0.44 | 16 | 478 | 41 | 0.0324 | 0.2807 | 0.0581 |\n",
      "0.45 | 15 | 458 | 42 | 0.0317 | 0.2632 | 0.0566 |\n",
      "0.46 | 15 | 436 | 42 | 0.0333 | 0.2632 | 0.0591 |\n",
      "0.47 | 15 | 414 | 42 | 0.0350 | 0.2632 | 0.0617 |\n",
      "0.48 | 15 | 381 | 42 | 0.0379 | 0.2632 | 0.0662 |\n",
      "0.49 | 15 | 364 | 42 | 0.0396 | 0.2632 | 0.0688 |\n",
      "0.50 | 15 | 348 | 42 | 0.0413 | 0.2632 | 0.0714 |\n",
      "0.51 | 15 | 335 | 42 | 0.0429 | 0.2632 | 0.0737 |\n",
      "0.52 | 15 | 318 | 42 | 0.0450 | 0.2632 | 0.0769 |\n",
      "0.53 | 15 | 294 | 42 | 0.0485 | 0.2632 | 0.0820 |\n",
      "0.54 | 15 | 282 | 42 | 0.0505 | 0.2632 | 0.0847 |\n",
      "0.55 | 15 | 269 | 42 | 0.0528 | 0.2632 | 0.0880 |\n",
      "0.56 | 15 | 256 | 42 | 0.0554 | 0.2632 | 0.0915 |\n",
      "0.57 | 14 | 241 | 43 | 0.0549 | 0.2456 | 0.0897 |\n",
      "0.58 | 14 | 229 | 43 | 0.0576 | 0.2456 | 0.0933 |\n",
      "0.59 | 12 | 214 | 45 | 0.0531 | 0.2105 | 0.0848 |\n",
      "0.60 | 12 | 204 | 45 | 0.0556 | 0.2105 | 0.0879 |\n",
      "0.61 | 11 | 196 | 46 | 0.0531 | 0.1930 | 0.0833 |\n",
      "0.62 | 11 | 186 | 46 | 0.0558 | 0.1930 | 0.0866 |\n",
      "0.63 | 10 | 178 | 47 | 0.0532 | 0.1754 | 0.0816 |\n",
      "0.64 | 10 | 169 | 47 | 0.0559 | 0.1754 | 0.0847 |\n",
      "0.65 | 10 | 161 | 47 | 0.0585 | 0.1754 | 0.0877 |\n",
      "0.66 | 10 | 153 | 47 | 0.0613 | 0.1754 | 0.0909 |\n",
      "0.67 | 10 | 145 | 47 | 0.0645 | 0.1754 | 0.0943 |\n",
      "0.68 | 10 | 135 | 47 | 0.0690 | 0.1754 | 0.0990 |\n",
      "0.69 | 10 | 126 | 47 | 0.0735 | 0.1754 | 0.1036 |\n",
      "0.70 | 10 | 122 | 47 | 0.0758 | 0.1754 | 0.1058 |\n",
      "0.71 | 10 | 118 | 47 | 0.0781 | 0.1754 | 0.1081 |\n",
      "0.72 | 10 | 110 | 47 | 0.0833 | 0.1754 | 0.1130 |\n",
      "0.73 | 10 | 103 | 47 | 0.0885 | 0.1754 | 0.1176 |\n",
      "0.74 | 9 | 98 | 48 | 0.0841 | 0.1579 | 0.1098 |\n",
      "0.75 | 9 | 92 | 48 | 0.0891 | 0.1579 | 0.1139 |\n",
      "0.76 | 9 | 89 | 48 | 0.0918 | 0.1579 | 0.1161 |\n",
      "0.77 | 9 | 84 | 48 | 0.0968 | 0.1579 | 0.1200 |\n",
      "0.78 | 8 | 81 | 49 | 0.0899 | 0.1404 | 0.1096 |\n",
      "0.79 | 8 | 78 | 49 | 0.0930 | 0.1404 | 0.1119 |\n",
      "0.80 | 8 | 74 | 49 | 0.0976 | 0.1404 | 0.1151 |\n",
      "0.81 | 8 | 70 | 49 | 0.1026 | 0.1404 | 0.1185 |\n",
      "0.82 | 8 | 68 | 49 | 0.1053 | 0.1404 | 0.1203 |\n",
      "0.83 | 8 | 65 | 49 | 0.1096 | 0.1404 | 0.1231 |\n",
      "0.84 | 8 | 62 | 49 | 0.1143 | 0.1404 | 0.1260 |\n",
      "0.85 | 8 | 59 | 49 | 0.1194 | 0.1404 | 0.1290 |\n",
      "0.86 | 8 | 56 | 49 | 0.1250 | 0.1404 | 0.1322 |\n",
      "0.87 | 8 | 54 | 49 | 0.1290 | 0.1404 | 0.1345 |\n",
      "0.88 | 8 | 50 | 49 | 0.1379 | 0.1404 | 0.1391 |\n",
      "0.89 | 8 | 48 | 49 | 0.1429 | 0.1404 | 0.1416 |\n",
      "0.90 | 7 | 46 | 50 | 0.1321 | 0.1228 | 0.1273 |\n",
      "0.91 | 7 | 45 | 50 | 0.1346 | 0.1228 | 0.1284 |\n",
      "0.92 | 6 | 44 | 51 | 0.1200 | 0.1053 | 0.1121 |\n",
      "0.93 | 6 | 43 | 51 | 0.1224 | 0.1053 | 0.1132 |\n",
      "0.94 | 6 | 41 | 51 | 0.1277 | 0.1053 | 0.1154 |\n",
      "0.95 | 5 | 37 | 52 | 0.1190 | 0.0877 | 0.1010 |\n",
      "0.96 | 5 | 35 | 52 | 0.1250 | 0.0877 | 0.1031 |\n",
      "0.97 | 4 | 34 | 53 | 0.1053 | 0.0702 | 0.0842 |\n",
      "0.98 | 4 | 34 | 53 | 0.1053 | 0.0702 | 0.0842 |\n",
      "0.99 | 4 | 34 | 53 | 0.1053 | 0.0702 | 0.0842 |\n"
     ]
    }
   ],
   "source": [
    "ablation_dict = functions.ablation_study_p_dev(\n",
    "    preds,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    clean_videos_images,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_code,\n",
    "    k,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6696e3028ae55fa5be357812587f73779dfb157cc851dab91c4ca8ffd3c7a806"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}