casme2_mae_main_dev.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A jupyter notebook version file for the `main.py`\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set `autoreload` to execute the change in `.py` files.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import _pickle\n",
    "import natsort\n",
    "from pathlib import Path\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "\n",
    "from __utils__ import image_processing\n",
    "from __utils__ import label_processing\n",
    "from __utils__ import labeling\n",
    "from __utils__ import loso_preparing\n",
    "from __utils__ import functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preds_path: D:\\Databases\\CAS(ME)^2\\preds\\mae_sl_swin_t_batch_size_48_epochs_25_pseudo_labeling_128.pkl\n"
     ]
    }
   ],
   "source": [
    "dataset_dir = \"D:/Databases/CAS(ME)^2\"\n",
    "# dataset_dir = \"F:/HEH/Databases/CAS(ME)^2\"\n",
    "# dataset_dir = \"/data/disk1/heh/databases/CAS(ME)^2\"\n",
    "\n",
    "images_loading = False\n",
    "image_size = 128\n",
    "load_cropped_images = False\n",
    "expression_type = \"mae\"  # macro-expression spotting\n",
    "# expression_type = \"me\" # micro-expression spotting\n",
    "debug_preds = True\n",
    "labeling_function = \"pseudo_labeling\"\n",
    "# labeling_function = \"original_labeling\"\n",
    "model_names = {\n",
    "    0: \"SOFTNet\",\n",
    "    1: \"SOFTNetCBAM\",\n",
    "    2: \"ViT-B\",\n",
    "    3: \"SL-ViT-B\",\n",
    "    4: \"Swin-T\",\n",
    "    5: \"Swin-S\",\n",
    "    6: \"L-Swin-T\",\n",
    "    7: \"S-Swin-T\",\n",
    "    8: \"SL-Swin-T\",\n",
    "    9: \"SL-Swin-S\",\n",
    "}\n",
    "model_name = model_names[8]\n",
    "batch_size = 48\n",
    "epochs = 25\n",
    "save_preds = False\n",
    "preds_stem = (\n",
    "    f\"{expression_type}_\"\n",
    "    + model_name.lower().replace(\"-\", \"_\")\n",
    "    + f\"_batch_size_{batch_size}\"\n",
    "    + f\"_epochs_{epochs}\"\n",
    "    + f\"_{labeling_function}\"\n",
    "    + f\"_{image_size}\"\n",
    "    # + \"_3\"\n",
    ")\n",
    "preds_path = Path(dataset_dir, \"preds\", preds_stem).with_suffix(\".pkl\")\n",
    "print(f\"preds_path: {preds_path}\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Images\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When debug the image processing, the videos_images is from cropped_rawpic, whereas the other variables are from rawpic.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subject:  s15\n",
      "subject:  s16\n",
      "subject:  s19\n",
      "subject:  s20\n",
      "subject:  s21\n",
      "subject:  s22\n",
      "subject:  s23\n",
      "subject:  s24\n",
      "subject:  s25\n",
      "subject:  s26\n",
      "subject:  s27\n",
      "subject:  s29\n",
      "subject:  s30\n",
      "subject:  s31\n",
      "subject:  s32\n",
      "subject:  s33\n",
      "subject:  s34\n",
      "subject:  s35\n",
      "subject:  s36\n",
      "subject:  s37\n",
      "subject:  s38\n",
      "subject:  s40\n"
     ]
    }
   ],
   "source": [
    "videos_images, subjects, subjects_videos_code = image_processing.load_images(\n",
    "    dataset_dir,\n",
    "    images_loading=images_loading,\n",
    "    image_size=image_size,\n",
    "    load_cropped_images=load_cropped_images,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subjects: ['s15', 's16', 's19', 's20', 's21', 's22', 's23', 's24', 's25', 's26', 's27', 's29', 's30', 's31', 's32', 's33', 's34', 's35', 's36', 's37', 's38', 's40']\n",
      "subjects_videos_code: [['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0508'], ['0101', '0102', '0401', '0402', '0502', '0505', '0507'], ['0102', '0402', '0505', '0507', '0502'], ['0502'], ['0101', '0401'], ['0101', '0102', '0402', '0503', '0508'], ['0102', '0402', '0503', '0507'], ['0101', '0401', '0402', '0502', '0507'], ['0101', '0102', '0502', '0508'], ['0101', '0102', '0401', '0503'], ['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0507', '0508'], ['0502'], ['0101', '0102', '0401', '0502', '0503', '0505', '0507'], ['0101', '0401', '0402', '0502', '0503', '0505', '0507'], ['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0507', '0508'], ['0102', '0402'], ['0401', '0402', '0503'], ['0102'], ['0401', '0505'], ['0101', '0402', '0502', '0505', '0507', '0508'], ['0502', '0507'], ['0401', '0502', '0503']]\n"
     ]
    }
   ],
   "source": [
    "print(\"subjects:\", subjects)\n",
    "print(\"subjects_videos_code:\", subjects_videos_code)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Excel \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participant</th>\n",
       "      <th>video_name_&amp;_expression_number</th>\n",
       "      <th>onset</th>\n",
       "      <th>apex</th>\n",
       "      <th>offset</th>\n",
       "      <th>AUs</th>\n",
       "      <th>extimated_emotion</th>\n",
       "      <th>expression_type</th>\n",
       "      <th>self-reported_emotion</th>\n",
       "      <th>video_name</th>\n",
       "      <th>video_code</th>\n",
       "      <th>subject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>anger1_1</td>\n",
       "      <td>557</td>\n",
       "      <td>572</td>\n",
       "      <td>608</td>\n",
       "      <td>4+10+14+15</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger1</td>\n",
       "      <td>0401</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>anger1_2</td>\n",
       "      <td>2854</td>\n",
       "      <td>2862</td>\n",
       "      <td>2871</td>\n",
       "      <td>38</td>\n",
       "      <td>others</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>sadness</td>\n",
       "      <td>anger1</td>\n",
       "      <td>0401</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>anger2_1</td>\n",
       "      <td>2155</td>\n",
       "      <td>2163</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger2</td>\n",
       "      <td>0402</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>anger2_2</td>\n",
       "      <td>3363</td>\n",
       "      <td>3371</td>\n",
       "      <td>3383</td>\n",
       "      <td>4+7+14</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger2</td>\n",
       "      <td>0402</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>anger2_3</td>\n",
       "      <td>3380</td>\n",
       "      <td>3386</td>\n",
       "      <td>3407</td>\n",
       "      <td>4+14+38</td>\n",
       "      <td>negative</td>\n",
       "      <td>macro-expression</td>\n",
       "      <td>anger</td>\n",
       "      <td>anger2</td>\n",
       "      <td>0402</td>\n",
       "      <td>s15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   participant video_name_&_expression_number  onset  apex  offset   \n",
       "0            1                       anger1_1    557   572     608  \\\n",
       "1            1                       anger1_2   2854  2862    2871   \n",
       "2            1                       anger2_1   2155  2163       0   \n",
       "3            1                       anger2_2   3363  3371    3383   \n",
       "4            1                       anger2_3   3380  3386    3407   \n",
       "\n",
       "          AUs extimated_emotion   expression_type self-reported_emotion   \n",
       "0  4+10+14+15          negative  macro-expression                 anger  \\\n",
       "1          38            others  macro-expression               sadness   \n",
       "2         NaN          negative  macro-expression                 anger   \n",
       "3      4+7+14          negative  macro-expression                 anger   \n",
       "4     4+14+38          negative  macro-expression                 anger   \n",
       "\n",
       "  video_name video_code subject  \n",
       "0     anger1       0401     s15  \n",
       "1     anger1       0401     s15  \n",
       "2     anger2       0402     s15  \n",
       "3     anger2       0402     s15  \n",
       "4     anger2       0402     s15  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Excel_data = label_processing.load_excel(dataset_dir)\n",
    "Excel_data.head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Ground Truth Labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "required_videos_index:  [0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96]\n",
      "len(clean_videos_images) = 88\n"
     ]
    }
   ],
   "source": [
    "(\n",
    "    clean_videos_images,\n",
    "    clean_subjects_videos_code,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    ") = label_processing.load_ground_truth_labels(\n",
    "    dataset_dir,\n",
    "    expression_type,\n",
    "    videos_images,\n",
    "    subjects_videos_code,\n",
    "    subjects,\n",
    "    Excel_data,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(clean_subjects):  20\n",
      "clean_subjects:  ['s15' 's16' 's19' 's20' 's21' 's22' 's23' 's24' 's25' 's26' 's27' 's30'\n",
      " 's31' 's32' 's33' 's34' 's36' 's37' 's38' 's40']\n",
      "len(clean_subjects_videos_code):  20\n",
      "clean_subjects_videos_codes:  [['0101', '0102', '0401', '0402', '0502', '0503', '0505'], ['0101', '0102', '0401', '0402', '0502', '0505', '0507'], ['0102', '0505', '0507'], ['0502'], ['0101', '0401'], ['0101', '0102', '0402', '0503', '0508'], ['0102', '0402', '0507'], ['0101', '0401', '0402', '0502', '0507'], ['0102', '0502', '0508'], ['0101', '0102', '0401', '0503'], ['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0507', '0508'], ['0102', '0401', '0502', '0503', '0505', '0507'], ['0101', '0401', '0402', '0502', '0503', '0505', '0507'], ['0101', '0102', '0401', '0402', '0502', '0503', '0505', '0507'], ['0102', '0402'], ['0401', '0402', '0503'], ['0401', '0505'], ['0101', '0402', '0502', '0505', '0507', '0508'], ['0502', '0507'], ['0401', '0502', '0503']]\n",
      "len(clean_subjects_videos_ground_truth_labels):  20\n",
      "clean_subjects_videos_ground_truth_labels:  [[[[1194, 1265]], [[395, 429], [406, 434], [780, 801]], [[556, 607], [2853, 2870]], [[2154, 2162], [3362, 3382], [3379, 3406], [3737, 3821], [3760, 3778]], [[177, 189], [2202, 2226]], [[2456, 2509]], [[1402, 1435]]], [[[166, 198], [332, 345], [696, 724], [914, 964], [1227, 1283], [1791, 1835], [1793, 1841]], [[186, 239], [172, 247], [316, 399], [321, 355], [321, 358], [419, 444], [525, 542], [719, 740]], [[553, 616], [1655, 1699], [1740, 1779], [3579, 3598]], [[414, 456], [1381, 1415], [1539, 1576], [1864, 1910], [2148, 2209], [2711, 2751], [2904, 2936], [3408, 3460], [3931, 3942]], [[337, 372], [849, 893], [1097, 1147], [1404, 1440], [1469, 1522], [2192, 2219]], [[67, 88], [990, 996], [1134, 1204], [1351, 1387], [1594, 1614], [1609, 1677], [1901, 1951], [2238, 2294]], [[843, 869], [1570, 1632], [2337, 2410], [3168, 3214]]], [[[821, 864], [886, 925]], [[1754, 1807]], [[2976, 3038]]], [[[980, 998], [1124, 1160], [1402, 1498]]], [[[168, 190]], [[3378, 3410]]], [[[365, 392], [379, 390], [509, 524], [1561, 1579], [1572, 1577], [1644, 1649], [1705, 1727], [1751, 1767], [1821, 1854], [1914, 1940]], [[300, 319], [320, 336], [903, 911]], [[3245, 3269]], [[2757, 2773]], [[80, 114]]], [[[912, 939], [996, 1014]], [[3684, 3718], [3798, 3817], [3878, 3902]], [[2743, 2763]]], [[[1355, 1379], [1382, 1418]], [[354, 384], [433, 449], [1261, 1268], [1902, 1949], [2207, 2275], [2542, 2575], [3410, 3439]], [[3296, 3341], [3552, 3598]], [[851, 868], [1077, 1100], [1583, 1629], [1862, 1888]], [[831, 880], [1420, 1463], [1647, 1657], [1663, 1702], [1857, 1881], [2004, 2043], [2228, 2246], [2690, 2745], [3167, 3222]]], [[[469, 489]], [[840, 912], [944, 983]], [[122, 166], [188, 202]]], [[[1013, 1039]], [[307, 344], [427, 446]], [[574, 621]], [[407, 437], [1726, 1756]]], [[[416, 472], [889, 910], [1200, 1219], [1220, 1223], [1702, 1721]], [[313, 331], [438, 456], [451, 508], [552, 571], [794, 817], [1052, 1073]], [[1151, 1176], [1681, 1732], [2157, 2181], [2240, 2257]], [[2631, 2652], [3726, 3744]], [[168, 250], [328, 340], [557, 598], [1080, 1132], [1437, 1464], [2186, 2218]], [[358, 393], [2224, 2270]], [[1121, 1233], [1757, 1812], [1917, 1981]], [[900, 918], [2676, 2701], [3186, 3211]], [[178, 242]]], [[[336, 384], [760, 803]], [[394, 421], [1580, 1621], [2982, 3020]], [[166, 196], [330, 409], [845, 894], [1171, 1194], [1568, 1613], [1917, 1953]], [[1418, 1437], [1546, 1561]], [[1140, 1165], [2231, 2301]], [[1679, 1697], [3178, 3192]]], [[[381, 456], [554, 610], [725, 842], [835, 885], [1088, 1139], [1149, 1168], [1560, 1600], [1630, 1639]], [[514, 533], [2849, 2875], [3336, 3382]], [[862, 887], [2122, 2155], [2204, 2225], [2394, 2449], [3276, 3350], [3478, 3508]], [[1594, 1652]], [[2168, 2201]], [[2241, 2278]], [[767, 792], [833, 856], [1499, 1519], [1564, 1585], [2246, 2267], [3113, 3143]]], [[[884, 918], [1233, 1249]], [[779, 801]], [[301, 321], [343, 359], [405, 438], [561, 608], [873, 896], [1309, 1362], [1517, 1559], [1649, 1673], [1691, 1741], [2248, 2334], [2406, 2450], [2795, 2844], [2911, 2960], [3172, 3196]], [[1530, 1549], [2269, 2294], [2773, 2801], [3199, 3209], [3395, 3425], [3744, 3806], [3998, 4072]], [[138, 193], [324, 344], [403, 409], [857, 908], [1569, 1608], [2202, 2228]], [[257, 318], [348, 415], [655, 715], [728, 754], [747, 819], [1081, 1142], [1551, 1585], [1558, 1610], [1750, 1779], [2076, 2131], [2453, 2499], [2662, 2727]], [[1361, 1379], [1381, 1404], [1708, 1749], [2211, 2271]], [[1645, 1742], [2331, 2347]]], [[[55, 75], [107, 129]], [[3681, 3698], [4327, 4355]]], [[[1762, 1786]], [[2169, 2184]], [[989, 1041]]], [[[105, 144]], [[347, 372], [565, 581], [1073, 1094]]], [[[1288, 1309]], [[2782, 2809], [3044, 3065], [3175, 3199], [3396, 3412]], [[850, 874], [1857, 1867]], [[413, 422], [1166, 1181], [1467, 1508], [1510, 1540], [1940, 1973], [2228, 2268]], [[845, 860], [1570, 1679], [1750, 1761], [2019, 2073], [2340, 2399], [2687, 2751]], [[232, 254], [321, 332]]], [[[328, 341], [877, 921], [961, 980], [1188, 1196], [1781, 1840], [1868, 1881]], [[1241, 1262], [1376, 1421], [1663, 1702], [1774, 1808], [1869, 1910]]], [[[391, 414], [591, 612]], [[600, 612], [1597, 1612]], [[2691, 2706]]]]\n"
     ]
    }
   ],
   "source": [
    "print(\"len(clean_subjects): \", len(clean_subjects))\n",
    "print(\"clean_subjects: \", clean_subjects)\n",
    "print(\"len(clean_subjects_videos_code): \", len(clean_subjects_videos_code))\n",
    "print(\"clean_subjects_videos_codes: \", clean_subjects_videos_code)\n",
    "print(\n",
    "    \"len(clean_subjects_videos_ground_truth_labels): \",\n",
    "    len(clean_subjects_videos_ground_truth_labels),\n",
    ")\n",
    "print(\n",
    "    \"clean_subjects_videos_ground_truth_labels: \",\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate `k`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "k (Half of average length of expression) =  18\n"
     ]
    }
   ],
   "source": [
    "k = label_processing.calculate_k(clean_subjects_videos_ground_truth_labels)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Labeling\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total frames: 231999\n"
     ]
    }
   ],
   "source": [
    "if labeling_function == \"pseudo_labeling\":\n",
    "    labels = labeling.get_pseudo_labels(\n",
    "        clean_videos_images, clean_subjects_videos_ground_truth_labels, k\n",
    "    )\n",
    "elif labeling_function == \"original_labeling\":\n",
    "    labels = labeling.get_original_labels(\n",
    "        clean_videos_images, clean_subjects_videos_ground_truth_labels, k\n",
    "    )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare for LOSO\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frame Index for each subject:-\n",
      "\n",
      "subject s15 ( group = 0): 0 -> 18470\n",
      "subject s15 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  7\n",
      "\n",
      "subject s16 ( group = 1): 18470 -> 37392\n",
      "subject s16 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  14\n",
      "\n",
      "subject s19 ( group = 2): 37392 -> 43977\n",
      "subject s19 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  17\n",
      "\n",
      "subject s20 ( group = 3): 43977 -> 46233\n",
      "subject s20 has 1 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  18\n",
      "\n",
      "subject s21 ( group = 4): 46233 -> 51966\n",
      "subject s21 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  20\n",
      "\n",
      "subject s22 ( group = 5): 51966 -> 62840\n",
      "subject s22 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  25\n",
      "\n",
      "subject s23 ( group = 6): 62840 -> 71496\n",
      "subject s23 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  28\n",
      "\n",
      "subject s24 ( group = 7): 71496 -> 87084\n",
      "subject s24 has 5 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  33\n",
      "\n",
      "subject s25 ( group = 8): 87084 -> 91039\n",
      "subject s25 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  36\n",
      "\n",
      "subject s26 ( group = 9): 91039 -> 100614\n",
      "subject s26 has 4 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  40\n",
      "\n",
      "subject s27 ( group = 10): 100614 -> 122975\n",
      "subject s27 has 9 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  49\n",
      "\n",
      "subject s30 ( group = 11): 122975 -> 138303\n",
      "subject s30 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  55\n",
      "\n",
      "subject s31 ( group = 12): 138303 -> 158962\n",
      "subject s31 has 7 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  62\n",
      "\n",
      "subject s32 ( group = 13): 158962 -> 180687\n",
      "subject s32 has 8 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  70\n",
      "\n",
      "subject s33 ( group = 14): 180687 -> 186097\n",
      "subject s33 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  72\n",
      "\n",
      "subject s34 ( group = 15): 186097 -> 196940\n",
      "subject s34 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  75\n",
      "\n",
      "subject s36 ( group = 16): 196940 -> 202930\n",
      "subject s36 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  77\n",
      "\n",
      "subject s37 ( group = 17): 202930 -> 217745\n",
      "subject s37 has 6 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  83\n",
      "\n",
      "subject s38 ( group = 18): 217745 -> 223250\n",
      "subject s38 has 2 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  85\n",
      "\n",
      "subject s40 ( group = 19): 223250 -> 231999\n",
      "subject s40 has 3 clean video(s)\n",
      "sum clean_subject_videos_ground_truth_labels_len:  88\n"
     ]
    }
   ],
   "source": [
    "y, groups = loso_preparing.prepare_for_loso(\n",
    "    labels,\n",
    "    clean_subjects,\n",
    "    clean_videos_images,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    k,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "if debug_preds is False:\n",
    "    from __utils__.training_dev import train\n",
    "\n",
    "    preds = train(\n",
    "        dataset_dir=dataset_dir,\n",
    "        clean_subjects=clean_subjects,\n",
    "        image_size=image_size,\n",
    "        y=y,\n",
    "        expression_type=expression_type,\n",
    "        model_name=model_name,\n",
    "        train_or_not=True,\n",
    "        batch_size=batch_size,\n",
    "        epochs=epochs,\n",
    "        preds_path=preds_path,\n",
    "    )\n",
    "else:\n",
    "    with open(preds_path, \"rb\") as pkl_file:\n",
    "        preds = _pickle.load(pkl_file)\n",
    "        pkl_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "if save_preds is True:\n",
    "    with open(preds_path, \"wb\") as pkl_file:\n",
    "        _pickle.dump(preds, pkl_file)\n",
    "        pkl_file.close()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spotting and Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split 1/20 is in process.\n",
      "0 video(s) have been processed.\n",
      "The current video be processed: subject s15, video 0101\n",
      "The current video be processed: subject s15, video 0102\n",
      "The current video be processed: subject s15, video 0401\n",
      "The current video be processed: subject s15, video 0402\n",
      "The current video be processed: subject s15, video 0502\n",
      "The current video be processed: subject s15, video 0503\n",
      "The current video be processed: subject s15, video 0505\n",
      "\n",
      "True Positive: 4, False Posive: 21, False Negative: 11\n",
      "Precision = 0.16000000000000023, Recall =0.2666666666666669, F1-Score = 0.20000000000000048\n",
      "Split 1/20 is processed.\n",
      "\n",
      "Split 2/20 is in process.\n",
      "7 video(s) have been processed.\n",
      "The current video be processed: subject s16, video 0101\n",
      "The current video be processed: subject s16, video 0102\n",
      "The current video be processed: subject s16, video 0401\n",
      "The current video be processed: subject s16, video 0402\n",
      "The current video be processed: subject s16, video 0502\n",
      "The current video be processed: subject s16, video 0505\n",
      "The current video be processed: subject s16, video 0507\n",
      "\n",
      "True Positive: 9, False Posive: 42, False Negative: 52\n",
      "Precision = 0.17647058823529435, Recall =0.1475409836065576, F1-Score = 0.16071428571428617\n",
      "Split 2/20 is processed.\n",
      "\n",
      "Split 3/20 is in process.\n",
      "14 video(s) have been processed.\n",
      "The current video be processed: subject s19, video 0102\n",
      "The current video be processed: subject s19, video 0505\n",
      "The current video be processed: subject s19, video 0507\n",
      "\n",
      "True Positive: 13, False Posive: 46, False Negative: 52\n",
      "Precision = 0.22033898305084768, Recall =0.20000000000000023, F1-Score = 0.20967741935483916\n",
      "Split 3/20 is processed.\n",
      "\n",
      "Split 4/20 is in process.\n",
      "17 video(s) have been processed.\n",
      "The current video be processed: subject s20, video 0502\n",
      "\n",
      "True Positive: 13, False Posive: 50, False Negative: 55\n",
      "Precision = 0.20634920634920656, Recall =0.1911764705882355, F1-Score = 0.1984732824427485\n",
      "Split 4/20 is processed.\n",
      "\n",
      "Split 5/20 is in process.\n",
      "18 video(s) have been processed.\n",
      "The current video be processed: subject s21, video 0101\n",
      "The current video be processed: subject s21, video 0401\n",
      "\n",
      "True Positive: 13, False Posive: 58, False Negative: 57\n",
      "Precision = 0.183098591549296, Recall =0.18571428571428594, F1-Score = 0.18439716312056784\n",
      "Split 5/20 is processed.\n",
      "\n",
      "Split 6/20 is in process.\n",
      "20 video(s) have been processed.\n",
      "The current video be processed: subject s22, video 0101\n",
      "The current video be processed: subject s22, video 0102\n",
      "The current video be processed: subject s22, video 0402\n",
      "The current video be processed: subject s22, video 0503\n",
      "The current video be processed: subject s22, video 0508\n",
      "\n",
      "True Positive: 16, False Posive: 63, False Negative: 70\n",
      "Precision = 0.20253164556962047, Recall =0.1860465116279072, F1-Score = 0.1939393939393944\n",
      "Split 6/20 is processed.\n",
      "\n",
      "Split 7/20 is in process.\n",
      "25 video(s) have been processed.\n",
      "The current video be processed: subject s23, video 0102\n",
      "The current video be processed: subject s23, video 0402\n",
      "The current video be processed: subject s23, video 0507\n",
      "\n",
      "True Positive: 19, False Posive: 76, False Negative: 73\n",
      "Precision = 0.20000000000000023, Recall =0.206521739130435, F1-Score = 0.2032085561497331\n",
      "Split 7/20 is processed.\n",
      "\n",
      "Split 8/20 is in process.\n",
      "28 video(s) have been processed.\n",
      "The current video be processed: subject s24, video 0101\n",
      "The current video be processed: subject s24, video 0401\n",
      "The current video be processed: subject s24, video 0402\n",
      "The current video be processed: subject s24, video 0502\n",
      "The current video be processed: subject s24, video 0507\n",
      "\n",
      "True Positive: 26, False Posive: 92, False Negative: 90\n",
      "Precision = 0.22033898305084768, Recall =0.22413793103448298, F1-Score = 0.22222222222222268\n",
      "Split 8/20 is processed.\n",
      "\n",
      "Split 9/20 is in process.\n",
      "33 video(s) have been processed.\n",
      "The current video be processed: subject s25, video 0102\n",
      "The current video be processed: subject s25, video 0502\n",
      "The current video be processed: subject s25, video 0508\n",
      "\n",
      "True Positive: 28, False Posive: 107, False Negative: 93\n",
      "Precision = 0.20740740740740762, Recall =0.23140495867768618, F1-Score = 0.21875000000000044\n",
      "Split 9/20 is processed.\n",
      "\n",
      "Split 10/20 is in process.\n",
      "36 video(s) have been processed.\n",
      "The current video be processed: subject s26, video 0101\n",
      "The current video be processed: subject s26, video 0102\n",
      "The current video be processed: subject s26, video 0401\n",
      "The current video be processed: subject s26, video 0503\n",
      "\n",
      "True Positive: 31, False Posive: 117, False Negative: 96\n",
      "Precision = 0.20945945945945968, Recall =0.2440944881889766, F1-Score = 0.2254545454545459\n",
      "Split 10/20 is processed.\n",
      "\n",
      "Split 11/20 is in process.\n",
      "40 video(s) have been processed.\n",
      "The current video be processed: subject s27, video 0101\n",
      "The current video be processed: subject s27, video 0102\n",
      "The current video be processed: subject s27, video 0401\n",
      "The current video be processed: subject s27, video 0402\n",
      "The current video be processed: subject s27, video 0502\n",
      "The current video be processed: subject s27, video 0503\n",
      "The current video be processed: subject s27, video 0505\n",
      "The current video be processed: subject s27, video 0507\n",
      "The current video be processed: subject s27, video 0508\n",
      "\n",
      "True Positive: 43, False Posive: 142, False Negative: 116\n",
      "Precision = 0.23243243243243267, Recall =0.27044025157232726, F1-Score = 0.25000000000000044\n",
      "Split 11/20 is processed.\n",
      "\n",
      "Split 12/20 is in process.\n",
      "49 video(s) have been processed.\n",
      "The current video be processed: subject s30, video 0102\n",
      "The current video be processed: subject s30, video 0401\n",
      "The current video be processed: subject s30, video 0502\n",
      "The current video be processed: subject s30, video 0503\n",
      "The current video be processed: subject s30, video 0505\n",
      "The current video be processed: subject s30, video 0507\n",
      "\n",
      "True Positive: 44, False Posive: 154, False Negative: 132\n",
      "Precision = 0.22222222222222243, Recall =0.2500000000000002, F1-Score = 0.23529411764705926\n",
      "Split 12/20 is processed.\n",
      "\n",
      "Split 13/20 is in process.\n",
      "55 video(s) have been processed.\n",
      "The current video be processed: subject s31, video 0101\n",
      "The current video be processed: subject s31, video 0401\n",
      "The current video be processed: subject s31, video 0402\n",
      "The current video be processed: subject s31, video 0502\n",
      "The current video be processed: subject s31, video 0503\n",
      "The current video be processed: subject s31, video 0505\n",
      "The current video be processed: subject s31, video 0507\n",
      "\n",
      "True Positive: 52, False Posive: 181, False Negative: 150\n",
      "Precision = 0.2231759656652363, Recall =0.25742574257425765, F1-Score = 0.2390804597701154\n",
      "Split 13/20 is processed.\n",
      "\n",
      "Split 14/20 is in process.\n",
      "62 video(s) have been processed.\n",
      "The current video be processed: subject s32, video 0101\n",
      "The current video be processed: subject s32, video 0102\n",
      "The current video be processed: subject s32, video 0401\n",
      "The current video be processed: subject s32, video 0402\n",
      "The current video be processed: subject s32, video 0502\n",
      "The current video be processed: subject s32, video 0503\n",
      "The current video be processed: subject s32, video 0505\n",
      "The current video be processed: subject s32, video 0507\n",
      "\n",
      "True Positive: 61, False Posive: 196, False Negative: 189\n",
      "Precision = 0.23735408560311305, Recall =0.24400000000000022, F1-Score = 0.24063116370808724\n",
      "Split 14/20 is processed.\n",
      "\n",
      "Split 15/20 is in process.\n",
      "70 video(s) have been processed.\n",
      "The current video be processed: subject s33, video 0102\n",
      "The current video be processed: subject s33, video 0402\n",
      "\n",
      "True Positive: 61, False Posive: 203, False Negative: 193\n",
      "Precision = 0.23106060606060627, Recall =0.24015748031496084, F1-Score = 0.23552123552123597\n",
      "Split 15/20 is processed.\n",
      "\n",
      "Split 16/20 is in process.\n",
      "72 video(s) have been processed.\n",
      "The current video be processed: subject s34, video 0401\n",
      "The current video be processed: subject s34, video 0402\n",
      "The current video be processed: subject s34, video 0503\n",
      "\n",
      "True Positive: 61, False Posive: 219, False Negative: 196\n",
      "Precision = 0.21785714285714308, Recall =0.23735408560311305, F1-Score = 0.22718808193668574\n",
      "Split 16/20 is processed.\n",
      "\n",
      "Split 17/20 is in process.\n",
      "75 video(s) have been processed.\n",
      "The current video be processed: subject s36, video 0401\n",
      "The current video be processed: subject s36, video 0505\n",
      "\n",
      "True Positive: 62, False Posive: 229, False Negative: 199\n",
      "Precision = 0.21305841924398647, Recall =0.23754789272030674, F1-Score = 0.22463768115942073\n",
      "Split 17/20 is processed.\n",
      "\n",
      "Split 18/20 is in process.\n",
      "77 video(s) have been processed.\n",
      "The current video be processed: subject s37, video 0101\n",
      "The current video be processed: subject s37, video 0402\n",
      "The current video be processed: subject s37, video 0502\n",
      "The current video be processed: subject s37, video 0505\n",
      "The current video be processed: subject s37, video 0507\n",
      "The current video be processed: subject s37, video 0508\n",
      "\n",
      "True Positive: 68, False Posive: 245, False Negative: 214\n",
      "Precision = 0.2172523961661344, Recall =0.24113475177304985, F1-Score = 0.228571428571429\n",
      "Split 18/20 is processed.\n",
      "\n",
      "Split 19/20 is in process.\n",
      "83 video(s) have been processed.\n",
      "The current video be processed: subject s38, video 0502\n",
      "The current video be processed: subject s38, video 0507\n",
      "\n",
      "True Positive: 70, False Posive: 248, False Negative: 223\n",
      "Precision = 0.22012578616352224, Recall =0.23890784982935176, F1-Score = 0.22913256955810193\n",
      "Split 19/20 is processed.\n",
      "\n",
      "Split 20/20 is in process.\n",
      "85 video(s) have been processed.\n",
      "The current video be processed: subject s40, video 0401\n",
      "The current video be processed: subject s40, video 0502\n",
      "The current video be processed: subject s40, video 0503\n",
      "\n",
      "True Positive: 70, False Posive: 258, False Negative: 228\n",
      "Precision = 0.21341463414634168, Recall =0.23489932885906062, F1-Score = 0.22364217252396212\n",
      "Split 20/20 is processed.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metric_fn, result_dict = functions.spot_and_evaluate(\n",
    "    preds,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    clean_videos_images,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_code,\n",
    "    k,\n",
    "    p=0.60,\n",
    "    show_plot_or_not=False,\n",
    ")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Positive: 70, False Posive: 258, False Negative: 228\n",
      "Final Precision = 0.21341463414634168,\n",
      "Final Recall =0.23489932885906062,\n",
      "Final F1-Score = 0.22364217252396212\n",
      "\n",
      "Highest Precision = 0.23735408560311305,\n",
      "Highest Recall =0.27044025157232726,\n",
      "Highest F1-Score = 0.25000000000000044\n"
     ]
    }
   ],
   "source": [
    "functions.final_evaluate(metric_fn, result_dict)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value (tf) | Value (tf) | Value\n",
    "| --- | --- | --- | --- | --- | ---\n",
    "| model | 3D-CNN | SOFTNet | SOFTNet (generator) | SOFTNet (dev) | SOFTNet\n",
    "| epochs | | 10 | 20 | 20 (252 m) | 30 (193 m) \n",
    "| batch_size | | 48 | 48 | 48 | 48\n",
    "| learning_rate | | 0.0005 | 0.0005 | 0.0005 | 0.0005\n",
    "| distance | | | k | k | 2k |\n",
    "| True Positive | | 90 | 95 | 83 | 11\n",
    "| False Positive | | 357 | 357 | 316 | 320 \n",
    "| False Negative | | 210 | 203 | 215 | 287\n",
    "| Precision | | 0.2013 | 0.2102 | 0.2080 | 0.0332\n",
    "| Recall | | 0.3000 | 0.3188 | 0.2785 | 0.0369\n",
    "| F1-Score | 0.2145 | 0.2410 | 0.2533 | 0.2381 | 0.0349\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value |\n",
    "| --- | --- | --- | --- |\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T |\n",
    "| epochs | 20 ( m) | 25 ( m) | 30 ( m) |\n",
    "| batch_size | 32 | 32 | 32 |\n",
    "| p |  |  |  |\n",
    "| True Positive |  |  |  |\n",
    "| False Positive |  |  |  |\n",
    "| False Negative |  |  |  |\n",
    "| Precision |  |  |  |\n",
    "| Recall |  |  | |\n",
    "| F1-Score |  |  |  |\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value | Value\n",
    "| --- | --- | --- | --- | ---\n",
    "| model | SL-Swin-T | SL-Swin-T | SL-Swin-T_2 | SL-Swin-T_3\n",
    "| epochs | 20 (2518 m) | 25 (2967 m) | 25 (2967 m) | 25 (3045 m)\n",
    "| batch_size | 48 | 48 | 48 | 48\n",
    "| p | 0.57 | 0.60 |0.60 | 0.60\n",
    "| True Positive | 66 | 70 | 67 | 65\n",
    "| False Positive | 279 | 258 | 255 | 261\n",
    "| False Negative | 232 | 228 | 231 | 233\n",
    "| Precision | 0.1913 | 0.2134 | 0.2081 | 0.1993\n",
    "| Recall | 0.2215 | 0.2349 | 0.2248 | 0.2181\n",
    "| F1-Score | 0.2053 | 0.2236 | 0.2161 | 0.2083\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value | Value |  Value | Value\n",
    "| --- | --- | --- | --- | --- | --- | ---\n",
    "| model | SL-Swin-S | SL-Swin-S | SL-Swin-S | SL-Swin-S | SL-Swin-S | SL-Swin-S\n",
    "| epochs | 20 ( m) | 25 (7262 m) | 30 ( m) | 20 (3542 m) | 25 (5269 m) | 30 ( m)\n",
    "| batch_size | 32 | 32 | 32 | 48 | 48 | 48\n",
    "| p | 0.86 | 0.77 | | 0.59 | 0.57 | 0.55\n",
    "| True Positive |  | 49 |  | 69 | 70 | 13\n",
    "| False Positive |  | 152 |  | 261 | 269 | 289\n",
    "| False Negative |  | 249 |  | 229 | 228 | 44\n",
    "| Precision |  | 0.2437 |  | 0.2091 | 0.2064 | 0.0430\n",
    "| Recall |  | 0.1644 | | 0.2315 | 0.2348 | 0.2280\n",
    "| F1-Score |  | 0.1963 |  | 0.2197 | 0.2197 | 0.0724\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value | Value | Value | Value\n",
    "| --- | --- | --- | --- | --- | ---\n",
    "| model | ViT-B | SL-ViT-B | Swin-T | L-Swin-T | S-Swin-T\n",
    "| epochs | 25 (4768 m) | 25 (8585 m) | 25 ( m) | 25 (3624 m) | 25 (2416 m)\n",
    "| batch_size | 48 | 96 | 48 | 48 | 48\n",
    "| p | 0.60 | 0.70 | 0.57 | 0.51 | 0.49\n",
    "| True Positive | 95 | 58 | 69 | 83 | 87 | \n",
    "| False Positive | 501 | 204 | 287 | 317 | 355 | \n",
    "| False Negative | 203 | 240 | 229 | 215 | 211 | \n",
    "| Precision | 0.1594 | 0.2214 | 0.1938 | 0.2075 | 0.1968 | \n",
    "| Recall | 0.3188 | 0.1946 | 0.2315 | 0.2785 | 0.2919 | \n",
    "| F1-Score | 0.2125 | 0.2071 | 0.2110 | 0.2378 | 0.2351 | \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Original Labeling\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Parameters | Value | Value |\n",
    "| --- | --- | --- |\n",
    "| model | Swin-T | SL-Swin-T |\n",
    "| epochs | 25 | 25 |\n",
    "| batch_size | 48 | 48 |\n",
    "| p | 0.58 | 0.60 |\n",
    "| True Positive | 72 | 71 |\n",
    "| False Positive | 234 | 264 |\n",
    "| False Negative | 226 | 227 |\n",
    "| Precision | 0.2353 | 0.2119 |\n",
    "| Recall | 0.2416 | 0.2383 |\n",
    "| F1-Score | 0.2384 | 0.2243 |\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ablation Study\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " p | TP | FP | FN | Precision | Recall | F1-Score\n",
      "0.01 | 159 | 3149 | 139 | 0.0481 | 0.5336 | 0.0882 |\n",
      "0.02 | 157 | 2853 | 141 | 0.0522 | 0.5268 | 0.0949 |\n",
      "0.03 | 157 | 2626 | 141 | 0.0564 | 0.5268 | 0.1019 |\n",
      "0.04 | 156 | 2383 | 142 | 0.0614 | 0.5235 | 0.1100 |\n",
      "0.05 | 155 | 2213 | 143 | 0.0655 | 0.5201 | 0.1163 |\n",
      "0.06 | 154 | 2051 | 144 | 0.0698 | 0.5168 | 0.1231 |\n",
      "0.07 | 153 | 1905 | 145 | 0.0743 | 0.5134 | 0.1299 |\n",
      "0.08 | 150 | 1766 | 148 | 0.0783 | 0.5034 | 0.1355 |\n",
      "0.09 | 147 | 1667 | 151 | 0.0810 | 0.4933 | 0.1392 |\n",
      "0.10 | 146 | 1585 | 152 | 0.0843 | 0.4899 | 0.1439 |\n",
      "0.11 | 145 | 1496 | 153 | 0.0884 | 0.4866 | 0.1496 |\n",
      "0.12 | 144 | 1426 | 154 | 0.0917 | 0.4832 | 0.1542 |\n",
      "0.13 | 141 | 1348 | 157 | 0.0947 | 0.4732 | 0.1578 |\n",
      "0.14 | 141 | 1276 | 157 | 0.0995 | 0.4732 | 0.1644 |\n",
      "0.15 | 140 | 1219 | 158 | 0.1030 | 0.4698 | 0.1690 |\n",
      "0.16 | 139 | 1153 | 159 | 0.1076 | 0.4664 | 0.1748 |\n",
      "0.17 | 137 | 1101 | 161 | 0.1107 | 0.4597 | 0.1784 |\n",
      "0.18 | 137 | 1048 | 161 | 0.1156 | 0.4597 | 0.1848 |\n",
      "0.19 | 137 | 998 | 161 | 0.1207 | 0.4597 | 0.1912 |\n",
      "0.20 | 137 | 964 | 161 | 0.1244 | 0.4597 | 0.1959 |\n",
      "0.21 | 136 | 929 | 162 | 0.1277 | 0.4564 | 0.1996 |\n",
      "0.22 | 132 | 888 | 166 | 0.1294 | 0.4430 | 0.2003 |\n",
      "0.23 | 131 | 846 | 167 | 0.1341 | 0.4396 | 0.2055 |\n",
      "0.24 | 129 | 813 | 169 | 0.1369 | 0.4329 | 0.2081 |\n",
      "0.25 | 127 | 787 | 171 | 0.1389 | 0.4262 | 0.2096 |\n",
      "0.26 | 126 | 756 | 172 | 0.1429 | 0.4228 | 0.2136 |\n",
      "0.27 | 123 | 738 | 175 | 0.1429 | 0.4128 | 0.2123 |\n",
      "0.28 | 121 | 714 | 177 | 0.1449 | 0.4060 | 0.2136 |\n",
      "0.29 | 119 | 701 | 179 | 0.1451 | 0.3993 | 0.2129 |\n",
      "0.30 | 118 | 676 | 180 | 0.1486 | 0.3960 | 0.2161 |\n",
      "0.31 | 117 | 657 | 181 | 0.1512 | 0.3926 | 0.2183 |\n",
      "0.32 | 117 | 633 | 181 | 0.1560 | 0.3926 | 0.2233 |\n",
      "0.33 | 116 | 609 | 182 | 0.1600 | 0.3893 | 0.2268 |\n",
      "0.34 | 115 | 592 | 183 | 0.1627 | 0.3859 | 0.2289 |\n",
      "0.35 | 111 | 576 | 187 | 0.1616 | 0.3725 | 0.2254 |\n",
      "0.36 | 110 | 565 | 188 | 0.1630 | 0.3691 | 0.2261 |\n",
      "0.37 | 108 | 543 | 190 | 0.1659 | 0.3624 | 0.2276 |\n",
      "0.38 | 108 | 524 | 190 | 0.1709 | 0.3624 | 0.2323 |\n",
      "0.39 | 107 | 505 | 191 | 0.1748 | 0.3591 | 0.2352 |\n",
      "0.40 | 104 | 492 | 194 | 0.1745 | 0.3490 | 0.2327 |\n",
      "0.41 | 102 | 482 | 196 | 0.1747 | 0.3423 | 0.2313 |\n",
      "0.42 | 102 | 465 | 196 | 0.1799 | 0.3423 | 0.2358 |\n",
      "0.43 | 99 | 448 | 199 | 0.1810 | 0.3322 | 0.2343 |\n",
      "0.44 | 99 | 436 | 199 | 0.1850 | 0.3322 | 0.2377 |\n",
      "0.45 | 96 | 417 | 202 | 0.1871 | 0.3221 | 0.2367 |\n",
      "0.46 | 92 | 401 | 206 | 0.1866 | 0.3087 | 0.2326 |\n",
      "0.47 | 90 | 381 | 208 | 0.1911 | 0.3020 | 0.2341 |\n",
      "0.48 | 87 | 369 | 211 | 0.1908 | 0.2919 | 0.2308 |\n",
      "0.49 | 84 | 363 | 214 | 0.1879 | 0.2819 | 0.2255 |\n",
      "0.50 | 83 | 349 | 215 | 0.1921 | 0.2785 | 0.2274 |\n",
      "0.51 | 83 | 343 | 215 | 0.1948 | 0.2785 | 0.2293 |\n",
      "0.52 | 81 | 330 | 217 | 0.1971 | 0.2718 | 0.2285 |\n",
      "0.53 | 79 | 323 | 219 | 0.1965 | 0.2651 | 0.2257 |\n",
      "0.54 | 78 | 309 | 220 | 0.2016 | 0.2617 | 0.2277 |\n",
      "0.55 | 76 | 299 | 222 | 0.2027 | 0.2550 | 0.2259 |\n",
      "0.56 | 75 | 292 | 223 | 0.2044 | 0.2517 | 0.2256 |\n",
      "0.57 | 74 | 283 | 224 | 0.2073 | 0.2483 | 0.2260 |\n",
      "0.58 | 73 | 273 | 225 | 0.2110 | 0.2450 | 0.2267 |\n",
      "0.59 | 71 | 263 | 227 | 0.2126 | 0.2383 | 0.2247 |\n",
      "0.60 | 70 | 258 | 228 | 0.2134 | 0.2349 | 0.2236 |\n",
      "0.61 | 64 | 250 | 234 | 0.2038 | 0.2148 | 0.2092 |\n",
      "0.62 | 61 | 243 | 237 | 0.2007 | 0.2047 | 0.2027 |\n",
      "0.63 | 57 | 236 | 241 | 0.1945 | 0.1913 | 0.1929 |\n",
      "0.64 | 56 | 226 | 242 | 0.1986 | 0.1879 | 0.1931 |\n",
      "0.65 | 55 | 223 | 243 | 0.1978 | 0.1846 | 0.1910 |\n",
      "0.66 | 52 | 212 | 246 | 0.1970 | 0.1745 | 0.1851 |\n",
      "0.67 | 51 | 205 | 247 | 0.1992 | 0.1711 | 0.1841 |\n",
      "0.68 | 51 | 195 | 247 | 0.2073 | 0.1711 | 0.1875 |\n",
      "0.69 | 49 | 188 | 249 | 0.2068 | 0.1644 | 0.1832 |\n",
      "0.70 | 46 | 184 | 252 | 0.2000 | 0.1544 | 0.1742 |\n",
      "0.71 | 44 | 178 | 254 | 0.1982 | 0.1477 | 0.1692 |\n",
      "0.72 | 44 | 168 | 254 | 0.2075 | 0.1477 | 0.1725 |\n",
      "0.73 | 43 | 165 | 255 | 0.2067 | 0.1443 | 0.1700 |\n",
      "0.74 | 43 | 159 | 255 | 0.2129 | 0.1443 | 0.1720 |\n",
      "0.75 | 42 | 157 | 256 | 0.2111 | 0.1409 | 0.1690 |\n",
      "0.76 | 39 | 150 | 259 | 0.2063 | 0.1309 | 0.1602 |\n",
      "0.77 | 38 | 146 | 260 | 0.2065 | 0.1275 | 0.1577 |\n",
      "0.78 | 38 | 142 | 260 | 0.2111 | 0.1275 | 0.1590 |\n",
      "0.79 | 38 | 139 | 260 | 0.2147 | 0.1275 | 0.1600 |\n",
      "0.80 | 38 | 136 | 260 | 0.2184 | 0.1275 | 0.1610 |\n",
      "0.81 | 38 | 133 | 260 | 0.2222 | 0.1275 | 0.1620 |\n",
      "0.82 | 36 | 130 | 262 | 0.2169 | 0.1208 | 0.1552 |\n",
      "0.83 | 34 | 125 | 264 | 0.2138 | 0.1141 | 0.1488 |\n",
      "0.84 | 33 | 123 | 265 | 0.2115 | 0.1107 | 0.1454 |\n",
      "0.85 | 30 | 120 | 268 | 0.2000 | 0.1007 | 0.1339 |\n",
      "0.86 | 30 | 114 | 268 | 0.2083 | 0.1007 | 0.1357 |\n",
      "0.87 | 30 | 110 | 268 | 0.2143 | 0.1007 | 0.1370 |\n",
      "0.88 | 30 | 106 | 268 | 0.2206 | 0.1007 | 0.1382 |\n",
      "0.89 | 29 | 98 | 269 | 0.2283 | 0.0973 | 0.1365 |\n",
      "0.90 | 29 | 95 | 269 | 0.2339 | 0.0973 | 0.1374 |\n",
      "0.91 | 27 | 89 | 271 | 0.2328 | 0.0906 | 0.1304 |\n",
      "0.92 | 26 | 87 | 272 | 0.2301 | 0.0872 | 0.1265 |\n",
      "0.93 | 26 | 86 | 272 | 0.2321 | 0.0872 | 0.1268 |\n",
      "0.94 | 24 | 84 | 274 | 0.2222 | 0.0805 | 0.1182 |\n",
      "0.95 | 24 | 80 | 274 | 0.2308 | 0.0805 | 0.1194 |\n",
      "0.96 | 23 | 76 | 275 | 0.2323 | 0.0772 | 0.1159 |\n",
      "0.97 | 23 | 73 | 275 | 0.2396 | 0.0772 | 0.1168 |\n",
      "0.98 | 21 | 70 | 277 | 0.2308 | 0.0705 | 0.1080 |\n",
      "0.99 | 21 | 69 | 277 | 0.2333 | 0.0705 | 0.1082 |\n"
     ]
    }
   ],
   "source": [
    "ablation_dict = functions.ablation_study_p_dev(\n",
    "    preds,\n",
    "    clean_subjects_videos_ground_truth_labels,\n",
    "    clean_videos_images,\n",
    "    clean_subjects,\n",
    "    clean_subjects_videos_code,\n",
    "    k,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6696e3028ae55fa5be357812587f73779dfb157cc851dab91c4ca8ffd3c7a806"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}