src/ABAW/abaw3_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=0\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = '/home/HDD6TB/datasets/emotions/ABAW'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from PIL import Image\n",
    "import cv2\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from sklearn import svm\n",
    "from sklearn.ensemble import RandomForestClassifier,RandomForestRegressor\n",
    "from sklearn import svm,metrics,preprocessing\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score,roc_auc_score,average_precision_score\n",
    "import mord\n",
    "\n",
    "from collections import defaultdict\n",
    "import os\n",
    "import glob\n",
    "import random\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import random\n",
    "import csv  \n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_names(id):\n",
    "    name = \"\"\n",
    "    if id>=0 and id<10:\n",
    "        name = \"0000\" + str(id)\n",
    "    elif id>=10 and id<100:\n",
    "        name = \"000\" + str(id)\n",
    "    elif id>=100 and id<1000:\n",
    "        name = \"00\" + str(id)\n",
    "    elif id>=1000 and id<10000:\n",
    "        name = \"0\" + str(id)\n",
    "    else:\n",
    "        name = str(id)\n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_filenames=lambda x: int(os.path.splitext(x)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1\n"
     ]
    }
   ],
   "source": [
    "import pathlib\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.applications.resnet import preprocess_input\n",
    "from tensorflow.keras.layers import TimeDistributed, GRU, Dense, Dropout, Flatten, LSTM, Activation, MaxPooling2D\n",
    "from tensorflow.keras.regularizers import l2 as L2_reg\n",
    "from tensorflow.keras.layers import Conv2D, BatchNormalization, \\\n",
    "    MaxPool2D, GlobalMaxPool2D, Input, Masking, Conv3D, MaxPooling3D, GlobalMaxPool3D\n",
    "from tensorflow.keras.optimizers import SGD, Adam\n",
    "import tensorflow.keras.backend as K \n",
    "\n",
    "print(tf.__version__)\n",
    "from tensorflow.compat.v1.keras.backend import set_session \n",
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess=tf.compat.v1.Session(config=config)\n",
    "set_session(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Number of frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27-60-1280x720 2932 02933.jpg\n",
      "video39 10678 10671.jpg\n",
      "77-30-1280x720 5672 05673.jpg\n",
      "91-30-1920x1080 10921 10922.jpg\n",
      "360 3509 03473.jpg\n",
      "136 4487 04488.jpg\n",
      "197 1869 01786.jpg\n",
      "291 1830 01831.jpg\n",
      "16-30-1920x1080 5473 05474.jpg\n",
      "video2 2064 02055.jpg\n",
      "89-30-1080x1920 630 00631.jpg\n",
      "408 2978 02978.jpg\n",
      "110-30-270x480 9978 09979.jpg\n",
      "324 5614 05614.jpg\n",
      "14-30-1920x1080 5310 05311.jpg\n",
      "129-24-1280x720 6153 06154.jpg\n",
      "153 3677 03678.jpg\n",
      "314 2260 02260.jpg\n",
      "255 1691 01692.jpg\n",
      "284 4054 04054.jpg\n",
      "298 2836 02836.jpg\n",
      "206 4476 04475.jpg\n",
      "video84 525 00525.jpg\n",
      "387 2911 02911.jpg\n",
      "286 2677 02678.jpg\n",
      "55-25-1280x720 1505 01480.jpg\n",
      "336 4344 04344.jpg\n",
      "video52 10679 10671.jpg\n",
      "60-30-1920x1080 4564 04564.jpg\n",
      "120 3591 03592.jpg\n",
      "267 3841 03841.jpg\n",
      "121-24-1920x1080 10628 10566.jpg\n",
      "349 4054 04054.jpg\n",
      "46-30-484x360 2944 02945.jpg\n",
      "218 3849 03830.jpg\n",
      "video55 8419 08409.jpg\n",
      "368 2929 02929.jpg\n",
      "9-15-1920x1080 47434 47435.jpg\n",
      "384 3451 03451.jpg\n",
      "228 2155 02155.jpg\n",
      "video3 7801 07802.jpg\n",
      "106-30-720x1280 1404 01405.jpg\n",
      "64-24-640x360 4918 04919.jpg\n",
      "244 2321 02288.jpg\n",
      "28-30-1280x720-2 396 00397.jpg\n",
      "235 3682 03683.jpg\n",
      "video86_3 402 00402.jpg\n",
      "92-24-1920x1080 8163 08164.jpg\n",
      "376 3569 03570.jpg\n",
      "185 4052 04053.jpg\n",
      "157 2604 02605.jpg\n",
      "38-30-1920x1080 2431 02432.jpg\n",
      "video87 1822 01808.jpg\n",
      "57-25-426x240 300 00301.jpg\n",
      "4-30-1920x1080 8260 08261.jpg\n",
      "video86_2 886 00886.jpg\n",
      "video45_5 414 00415.jpg\n",
      "143 10102 10103.jpg\n",
      "114 2449 02450.jpg\n",
      "243 4120 04121.jpg\n",
      "133 460 00461.jpg\n",
      "video64 6100 06100.jpg\n",
      "125-25-1280x720 6841 06842.jpg\n",
      "video81 8433 07975.jpg\n",
      "274 3526 03526.jpg\n",
      "113 15020 15021.jpg\n",
      "video66 154 00145.jpg\n",
      "171 3976 03977.jpg\n",
      "446 2248 02248.jpg\n",
      "126-30-1080x1920 4594 04594.jpg\n",
      "168 2978 02979.jpg\n",
      "video33 7553 07538.jpg\n",
      "201 1360 01360.jpg\n",
      "video61 116 00105.jpg\n",
      "236 3673 03673.jpg\n",
      "119 4576 04578.jpg\n",
      "116 2517 02518.jpg\n",
      "82-25-854x480 5402 05403.jpg\n",
      "15-24-1920x1080 3592 03593.jpg\n",
      "87-25-1920x1080 1802 01803.jpg\n",
      "36-24-1280x720 5260 05186.jpg\n",
      "130-25-1280x720 8461 08462.jpg\n",
      "219 3891 03888.jpg\n",
      "123-25-1920x1080 27241 27204.jpg\n",
      "318 4107 04079.jpg\n",
      "133-30-1280x720 2403 02404.jpg\n",
      "26-60-1280x720 2065 02066.jpg\n",
      "266 2465 02465.jpg\n",
      "7-60-1920x1080 23148 23149.jpg\n",
      "70-30-720x1280 2783 02784.jpg\n",
      "261 2866 02866.jpg\n",
      "video9 3794 03773.jpg\n",
      "199 2917 02918.jpg\n",
      "282 3614 03614.jpg\n",
      "305 4578 04579.jpg\n",
      "5-60-1920x1080-4 4892 04893.jpg\n",
      "116-30-1280x720 6183 06184.jpg\n",
      "video91 710 00710.jpg\n",
      "19-24-1920x1080 6633 06634.jpg\n",
      "video21 6808 06797.jpg\n",
      "196 5453 05454.jpg\n",
      "10-60-1280x720 2501 02502.jpg\n",
      "385 2976 02976.jpg\n",
      "video8 5367 05368.jpg\n",
      "video44 7785 07778.jpg\n",
      "video83 2702 02702.jpg\n",
      "202 940 00940.jpg\n",
      "32-60-1920x1080 4459 04460.jpg\n",
      "video90 3570 03546.jpg\n",
      "323 2936 02936.jpg\n",
      "198 2284 02285.jpg\n",
      "78-30-960x720 4442 04443.jpg\n",
      "video63 274 00263.jpg\n",
      "359 3861 03861.jpg\n",
      "386 1751 01752.jpg\n",
      "373 2704 02704.jpg\n",
      "225 1462 01462.jpg\n",
      "48-30-720x1280 1592 01593.jpg\n",
      "382 2713 02713.jpg\n",
      "video41 3494 03466.jpg\n",
      "176 6415 06416.jpg\n",
      "165 4567 04545.jpg\n",
      "31-30-1920x1080 4722 04723.jpg\n",
      "24-30-1920x1080-1 1912 01913.jpg\n",
      "214 2555 02555.jpg\n",
      "434 5232 05233.jpg\n",
      "426 2779 02779.jpg\n",
      "449 3552 03553.jpg\n",
      "149 15658 15648.jpg\n",
      "307 3310 03311.jpg\n",
      "video56 10695 10695.jpg\n",
      "video35 3567 03568.jpg\n",
      "319 2137 02137.jpg\n",
      "video74 18189 18189.jpg\n",
      "video45_6 308 00309.jpg\n",
      "video5 2468 02460.jpg\n",
      "347 2924 02924.jpg\n",
      "138-30-1280x720 3331 03332.jpg\n",
      "312 5109 05092.jpg\n",
      "video15 4287 04288.jpg\n",
      "326 4717 04718.jpg\n",
      "207 3038 03040.jpg\n",
      "204 5925 05925.jpg\n",
      "135-24-1920x1080 9423 09423.jpg\n",
      "video27 16418 16410.jpg\n",
      "358 324 00324.jpg\n",
      "video53 7979 07980.jpg\n",
      "208 5159 05159.jpg\n",
      "39-25-424x240 4590 04591.jpg\n",
      "137 4786 04787.jpg\n",
      "video14 7585 07586.jpg\n",
      "227 4417 04417.jpg\n",
      "155 3113 03114.jpg\n",
      "112 5915 05710.jpg\n",
      "video89 2241 02241.jpg\n",
      "402 4422 04422.jpg\n",
      "video67 1849 01849.jpg\n",
      "video38 2944 02936.jpg\n",
      "105 10076 10077.jpg\n",
      "134 1789 01790.jpg\n",
      "100-29-1080x1920 2639 02640.jpg\n",
      "167 4717 04718.jpg\n",
      "277 5171 05171.jpg\n",
      "346 5693 05694.jpg\n",
      "128 6286 06287.jpg\n",
      "269 4697 04697.jpg\n",
      "425 5070 04951.jpg\n",
      "video24 5204 05205.jpg\n",
      "103 26617 26618.jpg\n",
      "415 2903 02903.jpg\n",
      "393 4916 04915.jpg\n",
      "203 3964 03965.jpg\n",
      "439 2308 02308.jpg\n",
      "450 9907 09907.jpg\n",
      "110 5504 05505.jpg\n",
      "365 1067 01067.jpg\n",
      "395 5153 05153.jpg\n",
      "75-30-960x720 1953 01954.jpg\n",
      "398 3269 03269.jpg\n",
      "375 2957 02957.jpg\n",
      "258 1464 01464.jpg\n",
      "147 19694 19681.jpg\n",
      "video23 8789 08783.jpg\n",
      "83-24-1920x1080 5343 05344.jpg\n",
      "video93 443 00443.jpg\n",
      "58-30-640x480 2876 02877.jpg\n",
      "429 1673 01674.jpg\n",
      "63-30-1920x1080 2412 02413.jpg\n",
      "video46 3172 03167.jpg\n",
      "115-30-1280x720 4081 04081.jpg\n",
      "187 4404 04405.jpg\n",
      "111 7246 07247.jpg\n",
      "video10_1 4256 04249.jpg\n",
      "124-30-720x1280 10494 10495.jpg\n",
      "210 3238 03238.jpg\n",
      "video36 2424 02417.jpg\n",
      "364 3318 03318.jpg\n",
      "81-30-576x360 3603 03588.jpg\n",
      "328 2714 02714.jpg\n",
      "122-60-1920x1080-3 9243 09244.jpg\n",
      "186 4165 04162.jpg\n",
      "video45_7 179 00180.jpg\n",
      "22-30-1920x1080 5335 05336.jpg\n",
      "175 2502 02503.jpg\n",
      "video13 10566 10567.jpg\n",
      "80-30-320x240 5673 05674.jpg\n",
      "video25 3644 03645.jpg\n",
      "388 4158 04159.jpg\n",
      "112-30-640x360 6002 06002.jpg\n",
      "video85 7946 07946.jpg\n",
      "96-30-1280x720 5603 05604.jpg\n",
      "video34 10281 10271.jpg\n",
      "293 2110 02110.jpg\n",
      "234 2919 02919.jpg\n",
      "353 2856 02856.jpg\n",
      "288 3199 03199.jpg\n",
      "292 4703 04703.jpg\n",
      "424 4432 04433.jpg\n",
      "53-30-360x480 1052 01053.jpg\n",
      "122-60-1920x1080-1 7862 07863.jpg\n",
      "128-24-1920x1080 3812 03813.jpg\n",
      "179 4450 04451.jpg\n",
      "315 3937 03937.jpg\n",
      "148 18144 18145.jpg\n",
      "181 5861 05862.jpg\n",
      "194 4536 04537.jpg\n",
      "video77 3004 03004.jpg\n",
      "video86_1 1771 01771.jpg\n",
      "37-30-1280x720 2706 02707.jpg\n",
      "268 4366 04366.jpg\n",
      "290 3162 03159.jpg\n",
      "71-30-1920x1080 8001 07992.jpg\n",
      "399 5199 05200.jpg\n",
      "video75 640 00640.jpg\n",
      "8-30-1280x720 9113 09114.jpg\n",
      "video72 128 00120.jpg\n",
      "102-30-640x360 2983 02979.jpg\n",
      "177 3126 03127.jpg\n",
      "144 15605 15606.jpg\n",
      "123 9776 09768.jpg\n",
      "374 1212 01212.jpg\n",
      "348 4444 04443.jpg\n",
      "video12 5579 05580.jpg\n",
      "video7 3377 03027.jpg\n",
      "322 2428 02429.jpg\n",
      "309 2498 02498.jpg\n",
      "107-30-640x480 73 00073.jpg\n",
      "189 3854 03855.jpg\n",
      "372 2042 02042.jpg\n",
      "245 2530 02530.jpg\n",
      "229 3562 03563.jpg\n",
      "61-24-1920x1080 9156 09157.jpg\n",
      "159 3458 03459.jpg\n",
      "240 6791 06791.jpg\n",
      "195 6117 06118.jpg\n",
      "394 2335 02335.jpg\n",
      "84-30-1920x1080 2583 02583.jpg\n",
      "256 3336 03336.jpg\n",
      "273 4619 04619.jpg\n",
      "35-30-1920x1080 20807 20790.jpg\n",
      "278 4286 04287.jpg\n",
      "17-24-1920x1080 5787 05788.jpg\n",
      "246 1345 01345.jpg\n",
      "257 4388 04389.jpg\n",
      "275 2447 02447.jpg\n",
      "220 5025 05025.jpg\n",
      "334 3398 03398.jpg\n",
      "131-30-1920x1080 8012 08013.jpg\n",
      "video1 14587 14566.jpg\n",
      "18-24-1920x1080 4830 04831.jpg\n",
      "226 4096 04096.jpg\n",
      "109-30-1280x720 1045 01046.jpg\n",
      "335 3278 03278.jpg\n",
      "99-30-720x720 1800 01801.jpg\n",
      "video92 2553 02494.jpg\n",
      "418 2906 02906.jpg\n",
      "389 4556 04557.jpg\n",
      "260 2404 02405.jpg\n",
      "355 1631 01632.jpg\n",
      "357 1373 01373.jpg\n",
      "28-30-1280x720-1 262 00263.jpg\n",
      "59-30-1280x720 7741 07742.jpg\n",
      "25-25-600x480 5743 05690.jpg\n",
      "211 2162 02162.jpg\n",
      "105-30-1280x720 1501 01502.jpg\n",
      "241 3512 03512.jpg\n",
      "video73 5951 05936.jpg\n",
      "2-30-640x360 19352 19353.jpg\n",
      "254 3386 03386.jpg\n",
      "107 5315 05316.jpg\n",
      "137-30-1920x1080 8552 08553.jpg\n",
      "285 2988 02988.jpg\n",
      "200 5039 05040.jpg\n",
      "221 5124 05124.jpg\n",
      "video26 11020 11021.jpg\n",
      "video45_4 1038 01039.jpg\n",
      "172 11644 11645.jpg\n",
      "354 5389 05390.jpg\n",
      "391 3583 03583.jpg\n",
      "video60 1919 01906.jpg\n",
      "video82 10671 10671.jpg\n",
      "40-30-1280x720 9659 09660.jpg\n",
      "50-30-1920x1080 2699 02700.jpg\n",
      "252 2756 02757.jpg\n",
      "381 1265 01265.jpg\n",
      "video29 11259 11249.jpg\n",
      "65-30-400x228 6667 06638.jpg\n",
      "108-15-640x480 478 00479.jpg\n",
      "447 3993 03992.jpg\n",
      "video45_1 812 00813.jpg\n",
      "video51 4407 04408.jpg\n",
      "118-30-640x480 2420 02420.jpg\n",
      "190 5426 05427.jpg\n",
      "106 8163 08165.jpg\n",
      "294 3288 03288.jpg\n",
      "video54 3591 03591.jpg\n",
      "140 4582 04583.jpg\n",
      "141 4405 04406.jpg\n",
      "253 2276 02277.jpg\n",
      "127-30-1280x720 2213 02213.jpg\n",
      "247 2270 02270.jpg\n",
      "11-24-1920x1080 5389 05390.jpg\n",
      "289 4203 04203.jpg\n",
      "371 680 00680.jpg\n",
      "161 1728 01729.jpg\n",
      "47-30-654x480 4033 03921.jpg\n",
      "140-30-632x360 6754 06739.jpg\n",
      "317 1685 01685.jpg\n",
      "182 4355 04356.jpg\n",
      "85-24-1280x720 2664 02665.jpg\n",
      "345 2344 02345.jpg\n",
      "97-29-1920x1080 3366 03367.jpg\n",
      "video11 4379 04380.jpg\n",
      "45-24-1280x720 6081 06082.jpg\n",
      "212 4562 04562.jpg\n",
      "28-30-1280x720-4 276 00277.jpg\n",
      "251 2106 02106.jpg\n",
      "129 9013 09015.jpg\n",
      "120-30-1280x720 15041 15038.jpg\n",
      "30-30-1920x1080 3062 03063.jpg\n",
      "166 4514 04515.jpg\n",
      "56-30-1080x1920 4300 04301.jpg\n",
      "295 3769 03769.jpg\n",
      "video20 11271 11272.jpg\n",
      "320 4738 04618.jpg\n",
      "5-60-1920x1080-1 6153 06154.jpg\n",
      "117-25-1920x1080 4081 04078.jpg\n",
      "403 2165 02165.jpg\n",
      "440 2520 02520.jpg\n",
      "213 6397 06397.jpg\n",
      "video88 8850 08829.jpg\n",
      "232 2372 02372.jpg\n",
      "104-17-720x480 2425 02426.jpg\n",
      "29-24-1280x720 3683 03684.jpg\n",
      "224 2233 02140.jpg\n",
      "331 2954 02955.jpg\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "264 4744 04745.jpg\n",
      "video30 8217 08214.jpg\n",
      "341 3096 03098.jpg\n",
      "215 3409 03378.jpg\n",
      "94-30-1920x1080 2101 02102.jpg\n",
      "421 3615 03615.jpg\n",
      "138 5583 05584.jpg\n",
      "117 5465 05466.jpg\n",
      "video42 9226 09217.jpg\n",
      "6-30-1920x1080 7985 07986.jpg\n",
      "332 4665 04560.jpg\n",
      "409 3608 03608.jpg\n",
      "44-25-426x240 300 00301.jpg\n",
      "154 3362 03363.jpg\n",
      "407 3817 03817.jpg\n",
      "41-24-1280x720 6160 06161.jpg\n",
      "5-60-1920x1080-3 6333 06334.jpg\n",
      "158 3031 03032.jpg\n",
      "43-30-406x720 2873 02874.jpg\n",
      "video94 696 00696.jpg\n",
      "392 3550 03550.jpg\n",
      "video19 784 00775.jpg\n",
      "183 2769 02770.jpg\n",
      "video6 5324 05325.jpg\n",
      "video65 4571 04572.jpg\n",
      "250 2928 02928.jpg\n",
      "5-60-1920x1080-2 6213 06214.jpg\n",
      "3-25-1920x1080 6188 06189.jpg\n",
      "363 3240 03240.jpg\n",
      "233 4479 04360.jpg\n",
      "52-30-1280x720 1924 01863.jpg\n",
      "299 2336 02337.jpg\n",
      "135 3591 03593.jpg\n",
      "329 5086 05087.jpg\n",
      "video37 9186 09116.jpg\n",
      "video62 5579 05579.jpg\n",
      "423 5265 05265.jpg\n",
      "416 6511 06512.jpg\n",
      "122 4127 04128.jpg\n",
      "21-24-1920x1080 5230 05231.jpg\n",
      "369 4438 04437.jpg\n",
      "video16 12486 12487.jpg\n",
      "223 2797 02797.jpg\n",
      "video48 2542 02534.jpg\n",
      "134-30-1280x720 3875 03875.jpg\n",
      "114-30-1280x720 3662 03662.jpg\n",
      "126 6286 06287.jpg\n",
      "54-30-1080x1920 6445 06446.jpg\n",
      "122-60-1920x1080-5 12065 12066.jpg\n",
      "209 2856 02738.jpg\n",
      "119-30-848x480 1651 01652.jpg\n",
      "146 4850 04852.jpg\n",
      "51-30-1280x720 2499 02500.jpg\n",
      "339 4632 04631.jpg\n",
      "72-30-1280x720 7571 07572.jpg\n",
      "23-24-1920x1080 5411 05412.jpg\n",
      "49-30-1280x720 847 00848.jpg\n",
      "98-30-360x360 6645 06646.jpg\n",
      "113-60-1280x720 4022 04022.jpg\n",
      "79-30-960x720 2494 02495.jpg\n",
      "162 7934 07935.jpg\n",
      "303 3950 03950.jpg\n",
      "378 3480 03480.jpg\n",
      "101-30-1080x1920 5153 05154.jpg\n",
      "283 4768 04648.jpg\n",
      "118 587 00588.jpg\n",
      "video22 10469 10458.jpg\n",
      "370 1939 01939.jpg\n",
      "131 4777 04779.jpg\n",
      "76-30-640x280 1773 01774.jpg\n",
      "111-25-1920x1080 771 00772.jpg\n",
      "132-30-426x240 992 00974.jpg\n",
      "419 6963 06961.jpg\n",
      "1-30-1280x720 10355 10356.jpg\n",
      "132 5659 05660.jpg\n",
      "12-24-1920x1080 5685 05686.jpg\n",
      "265 6719 06719.jpg\n",
      "367 4169 04169.jpg\n",
      "video4 1550 01541.jpg\n",
      "103-30-384x480 771 00772.jpg\n",
      "video76 4274 04159.jpg\n",
      "74-25-1920x1080 5041 05042.jpg\n",
      "20-24-1920x1080 5274 05275.jpg\n",
      "184 3790 03791.jpg\n",
      "231 4207 04208.jpg\n",
      "304 2693 02694.jpg\n",
      "video96 8046 08046.jpg\n",
      "130 4001 04002.jpg\n",
      "248 2267 02267.jpg\n",
      "380 2375 02375.jpg\n",
      "361 4602 04602.jpg\n",
      "video69 8622 08622.jpg\n",
      "230 3970 03970.jpg\n",
      "160 837 00838.jpg\n",
      "34-25-1920x1080 7757 07758.jpg\n",
      "297 3288 03288.jpg\n",
      "259 2393 02393.jpg\n",
      "125 4365 04366.jpg\n",
      "428 3229 03229.jpg\n",
      "262 4753 04753.jpg\n",
      "66-25-1080x1920 2913 02914.jpg\n",
      "271 2223 02224.jpg\n",
      "237 5434 05434.jpg\n",
      "video58 303 00293.jpg\n",
      "435 1950 01950.jpg\n",
      "441 2701 02701.jpg\n",
      "362 4912 04912.jpg\n",
      "121 4413 04414.jpg\n",
      "330 4568 04569.jpg\n",
      "327 4156 04155.jpg\n",
      "28-30-1280x720-3 276 00277.jpg\n",
      "video78 419 00419.jpg\n",
      "306 2296 02297.jpg\n",
      "88-30-360x480 3722 03702.jpg\n",
      "video80 419 00419.jpg\n",
      "13-30-1920x1080 7399 07400.jpg\n",
      "427 1222 01222.jpg\n",
      "139-14-720x480 10013 10014.jpg\n",
      "24-30-1920x1080-2 4172 04173.jpg\n",
      "86-24-1920x1080 4995 04987.jpg\n",
      "video32 2775 02758.jpg\n",
      "169 2862 02863.jpg\n",
      "287 1272 01273.jpg\n",
      "42-30-480x480 1394 01395.jpg\n",
      "321 3555 03555.jpg\n",
      "412 1473 01473.jpg\n",
      "281 7011 07011.jpg\n",
      "239 1964 01964.jpg\n",
      "127 8123 07988.jpg\n",
      "93-24-640x360 2581 02582.jpg\n",
      "video59 6702 06702.jpg\n",
      "69-25-854x480 18781 18782.jpg\n",
      "313 3139 03140.jpg\n",
      "136-30-1920x1080 7529 07482.jpg\n",
      "33-30-1920x1080 10806 10807.jpg\n",
      "68-24-1920x1080 7189 07190.jpg\n",
      "279 2212 02212.jpg\n",
      "156 4925 04926.jpg\n",
      "video95 10855 10855.jpg\n",
      "406 3066 03066.jpg\n",
      "308 4651 04651.jpg\n",
      "188 5986 05987.jpg\n",
      "67-24-640x360 3461 03462.jpg\n",
      "62-30-654x480 12033 12034.jpg\n",
      "122-60-1920x1080-2 7172 07173.jpg\n",
      "102 4777 04778.jpg\n",
      "420 3150 03150.jpg\n",
      "192 3598 03599.jpg\n",
      "video47 820 00812.jpg\n",
      "337 2806 02790.jpg\n",
      "video49 6121 06115.jpg\n",
      "433 4310 04311.jpg\n",
      "video17 21937 21938.jpg\n",
      "448 4157 04157.jpg\n",
      "325 3222 03222.jpg\n",
      "377 3444 03443.jpg\n",
      "249 2621 02621.jpg\n",
      "276 1583 01583.jpg\n",
      "383 2554 02554.jpg\n",
      "163 3937 03938.jpg\n",
      "164 7043 07044.jpg\n",
      "191 5248 05249.jpg\n",
      "193 3983 03984.jpg\n",
      "video45_2 1095 01096.jpg\n",
      "video18 3783 03784.jpg\n",
      "video71 11204 11204.jpg\n",
      "430 1834 01834.jpg\n",
      "242 7010 07010.jpg\n",
      "90-30-1080x1920 7413 07414.jpg\n",
      "video70 11868 11861.jpg\n",
      "video45_3 294 00295.jpg\n",
      "151 964 00965.jpg\n",
      "video79 4097 04086.jpg\n",
      "238 3012 03013.jpg\n",
      "178 5840 05841.jpg\n",
      "95-24-1920x1080 2883 02884.jpg\n",
      "video40 5876 05877.jpg\n",
      "311 1930 01930.jpg\n",
      "296 6486 06486.jpg\n",
      "150 13015 13016.jpg\n",
      "video28 8727 08728.jpg\n",
      "122-60-1920x1080-4 7683 07684.jpg\n",
      "272 3333 03333.jpg\n",
      "350 2015 02015.jpg\n",
      "216 457 00457.jpg\n",
      "108 4678 04680.jpg\n",
      "video57 8379 08331.jpg\n",
      "400 2640 02640.jpg\n",
      "344 4263 04263.jpg\n",
      "280 3945 03945.jpg\n",
      "139 8165 08166.jpg\n",
      "270 3937 03937.jpg\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "d=os.path.join(DATA_DIR,'videos')\n",
    "video2len={}\n",
    "for filename in os.listdir(d):\n",
    "    fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "    vid=os.path.join(d,filename)\n",
    "    cap = cv2.VideoCapture(vid)\n",
    "    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "    video2len[fn]=total_frames+1 #FIX ME!!! NEED TO ADD 1 to teh number of frames for consistency with challeng's organizer\n",
    "    \n",
    "    dn=os.path.join(data_dir,fn)\n",
    "    last_img=''\n",
    "    if os.path.exists(dn):\n",
    "        images=[img_name for img_name in os.listdir(dn) if img_name.lower().endswith('.jpg')]\n",
    "        last_img=sorted(images, key=compare_filenames)[-1]\n",
    "    elif os.path.exists(dn+'_left'):\n",
    "        images=[img_name for img_name in os.listdir(dn+'_left') if img_name.lower().endswith('.jpg')]\n",
    "        last_img=sorted(images, key=compare_filenames)[-1]\n",
    "    print(fn,total_frames,last_img)\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.applications import mobilenet\n",
    "from tensorflow.keras.models import load_model,Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mobilenet_preprocess_input(x,**kwargs):\n",
    "    x[..., 0] -= 103.939\n",
    "    x[..., 1] -= 116.779\n",
    "    x[..., 2] -= 123.68\n",
    "    return x\n",
    "\n",
    "preprocessing_function=mobilenet_preprocess_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:No training configuration found in the save file, so the model was *not* compiled. Compile it manually.\n",
      "224 224\n"
     ]
    }
   ],
   "source": [
    "model_name='mymobilenet_7_ft_sgd_model'\n",
    "num_classes=7\n",
    "\n",
    "base_model=load_model('../../models/affectnet_emotions/'+model_name+'.h5')\n",
    "_,w,h,_=base_model.input.shape\n",
    "print(w,h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<KerasTensor: shape=(None, 1024) dtype=float32 (created by layer 'global_pooling')>,\n",
       " <KerasTensor: shape=(None, 256) dtype=float32 (created by layer 'feats')>,\n",
       " <KerasTensor: shape=(None, 7) dtype=float32 (created by layer 'emotion_preds')>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_extractor_model=Model(base_model.input,[base_model.get_layer('global_pooling').output,base_model.get_layer('feats').output,base_model.output])\n",
    "feature_extractor_model.summary()\n",
    "print(feature_extractor_model.output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(256, 7) (7,)\n"
     ]
    }
   ],
   "source": [
    "weights,bias=base_model.get_layer('emotion_preds').get_weights()\n",
    "print(weights.shape,bias.shape)\n",
    "\n",
    "def get_probab(features, logits=False):\n",
    "    x=np.dot(features,weights)+bias\n",
    "    if logits:\n",
    "        return x\n",
    "    e_x = np.exp(x - np.max(x,axis=0))\n",
    "    return e_x / e_x.sum(axis=1)[:,None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/564 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/HDD6TB/datasets/emotions/ABAW/faces\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 564/564 [5:04:48<00:00, 32.43s/it]    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2658563 (2658563, 1024) (2658563, 7)\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "print(data_dir)\n",
    "img_names=[]\n",
    "X_global_features,X_scores=[],[]\n",
    "imgs=[]\n",
    "for filename in tqdm(os.listdir(data_dir)):\n",
    "    frames_dir=os.path.join(data_dir,filename)\n",
    "    for img_name in os.listdir(frames_dir):\n",
    "        if img_name.lower().endswith('.jpg'):\n",
    "            img=cv2.imread(os.path.join(frames_dir,img_name))\n",
    "            if img.size:\n",
    "                img_names.append(filename+'/'+img_name)\n",
    "                img=cv2.resize(img,(w,h))\n",
    "                img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n",
    "                imgs.append(img)\n",
    "                if len(imgs)>=512:        \n",
    "                    inp = preprocessing_function(np.array(imgs, dtype=np.float32))\n",
    "                    global_features,feats,scores=feature_extractor_model.predict(inp)\n",
    "                    if len(X_scores)==0:\n",
    "                        X_global_features=global_features\n",
    "                        X_scores=scores\n",
    "                    else:\n",
    "                        X_global_features=np.concatenate((X_global_features,global_features),axis=0)\n",
    "                        X_scores=np.concatenate((X_scores,scores),axis=0)\n",
    "                    imgs=[]\n",
    "\n",
    "if len(imgs)>0:        \n",
    "    inp = preprocessing_function(np.array(imgs, dtype=np.float32))\n",
    "    global_features,feats,scores=feature_extractor_model.predict(inp)\n",
    "    if len(X_scores)==0:\n",
    "        X_global_features=global_features\n",
    "        X_scores=scores\n",
    "    else:\n",
    "        X_global_features=np.concatenate((X_global_features,global_features),axis=0)\n",
    "        X_scores=np.concatenate((X_scores,scores),axis=0)\n",
    "\n",
    "print(len(img_names),X_global_features.shape,X_scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2658563\n"
     ]
    }
   ],
   "source": [
    "filename2featuresAll={img_name:(global_features,scores) for img_name,global_features,scores in zip(img_names,X_global_features,X_scores)}\n",
    "print(len(filename2featuresAll))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Torch: 1.7.1+cu110\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "import glob\n",
    "from itertools import chain\n",
    "import os\n",
    "import random\n",
    "import zipfile\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import train_test_split\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torchvision import datasets, transforms\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "print(f\"Torch: {torch.__version__}\")\n",
    "device = 'cuda'\n",
    "use_cuda = torch.cuda.is_available()\n",
    "print(use_cuda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "enet_b0_8_best_vgaf.pt\n"
     ]
    }
   ],
   "source": [
    "if False:\n",
    "    PATH='enet_b2_8.pt'\n",
    "    #PATH='enet_b2_7.pt'\n",
    "    IMG_SIZE=260\n",
    "else:\n",
    "    #PATH='enet_b0_7.pt'\n",
    "    #PATH='enet_b0_8_best_afew.pt'\n",
    "    PATH='enet_b0_8_best_vgaf.pt'\n",
    "    IMG_SIZE=224\n",
    "    \n",
    "test_transforms = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize((IMG_SIZE,IMG_SIZE)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                     std=[0.229, 0.224, 0.225])\n",
    "    ]\n",
    ")\n",
    "\n",
    "print(PATH)\n",
    "feature_extractor_model = torch.load('../../models/affectnet_emotions/'+PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8, 1280) [[-0.00668902  0.11335418  0.04948402 ...  0.06092974 -0.02686705\n",
      "  -0.2584354 ]\n",
      " [-0.05986994 -0.11609969 -0.11716661 ... -0.0897665   0.02036773\n",
      "   0.11924423]\n",
      " [ 0.05512754 -0.0534068  -0.001405   ...  0.08906588 -0.12142781\n",
      "   0.0115855 ]\n",
      " ...\n",
      " [-0.060075    0.05815501 -0.03156688 ... -0.0667517   0.09121078\n",
      "   0.06233402]\n",
      " [ 0.08181744  0.1318671   0.25012463 ...  0.03424729 -0.0969213\n",
      "   0.15267523]\n",
      " [ 0.10014281 -0.07073897 -0.13113672 ... -0.11299834  0.12007102\n",
      "  -0.12579045]]\n",
      "(8,) [-0.08768775  0.00056133 -0.0919608   0.00036013  0.1872526   0.14050813\n",
      " -0.09973471  0.01581638]\n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "    classifier_weights=feature_extractor_model.classifier[0].weight.cpu().data.numpy()\n",
    "    classifier_bias=feature_extractor_model.classifier[0].bias.cpu().data.numpy()\n",
    "else:\n",
    "    classifier_weights=feature_extractor_model.classifier.weight.cpu().data.numpy()\n",
    "    classifier_bias=feature_extractor_model.classifier.bias.cpu().data.numpy()\n",
    "print(classifier_weights.shape,classifier_weights)\n",
    "print(classifier_bias.shape,classifier_bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EfficientNet(\n",
       "  (conv_stem): Conv2dSame(3, 32, kernel_size=(3, 3), stride=(2, 2), bias=False)\n",
       "  (bn1): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (act1): SiLU(inplace=True)\n",
       "  (blocks): Sequential(\n",
       "    (0): Sequential(\n",
       "      (0): DepthwiseSeparableConv(\n",
       "        (conv_dw): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=32, bias=False)\n",
       "        (bn1): BatchNorm2d(32, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(32, 8, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(8, 32, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pw): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(16, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): Identity()\n",
       "      )\n",
       "    )\n",
       "    (1): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(16, 96, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(96, 96, kernel_size=(3, 3), stride=(2, 2), groups=96, bias=False)\n",
       "        (bn2): BatchNorm2d(96, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(96, 4, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(4, 96, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(96, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(24, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(144, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(144, 144, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=144, bias=False)\n",
       "        (bn2): BatchNorm2d(144, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(144, 24, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(24, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (2): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(144, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(144, 144, kernel_size=(5, 5), stride=(2, 2), groups=144, bias=False)\n",
       "        (bn2): BatchNorm2d(144, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(144, 6, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(6, 144, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(144, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(40, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(240, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(240, 240, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=240, bias=False)\n",
       "        (bn2): BatchNorm2d(240, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(240, 10, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(10, 240, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(40, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (3): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(240, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(240, 240, kernel_size=(3, 3), stride=(2, 2), groups=240, bias=False)\n",
       "        (bn2): BatchNorm2d(240, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(240, 10, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(10, 240, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(240, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(480, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=480, bias=False)\n",
       "        (bn2): BatchNorm2d(480, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(480, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(480, 480, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=480, bias=False)\n",
       "        (bn2): BatchNorm2d(480, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(480, 80, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(80, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (4): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(80, 480, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(480, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(480, 480, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=480, bias=False)\n",
       "        (bn2): BatchNorm2d(480, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(480, 20, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(20, 480, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(480, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(672, 672, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(112, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (5): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2dSame(672, 672, kernel_size=(5, 5), stride=(2, 2), groups=672, bias=False)\n",
       "        (bn2): BatchNorm2d(672, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(672, 28, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(28, 672, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(672, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): InvertedResidual(\n",
       "        (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)\n",
       "        (bn2): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): InvertedResidual(\n",
       "        (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)\n",
       "        (bn2): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): InvertedResidual(\n",
       "        (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1152, 1152, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2), groups=1152, bias=False)\n",
       "        (bn2): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1152, 192, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(192, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (6): Sequential(\n",
       "      (0): InvertedResidual(\n",
       "        (conv_pw): Conv2d(192, 1152, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act1): SiLU(inplace=True)\n",
       "        (conv_dw): Conv2d(1152, 1152, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1152, bias=False)\n",
       "        (bn2): BatchNorm2d(1152, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (act2): SiLU(inplace=True)\n",
       "        (se): SqueezeExcite(\n",
       "          (conv_reduce): Conv2d(1152, 48, kernel_size=(1, 1), stride=(1, 1))\n",
       "          (act1): SiLU(inplace=True)\n",
       "          (conv_expand): Conv2d(48, 1152, kernel_size=(1, 1), stride=(1, 1))\n",
       "        )\n",
       "        (conv_pwl): Conv2d(1152, 320, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (bn3): BatchNorm2d(320, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (conv_head): Conv2d(320, 1280, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "  (bn2): BatchNorm2d(1280, eps=0.001, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (act2): SiLU(inplace=True)\n",
       "  (global_pool): SelectAdaptivePool2d (pool_type=avg, flatten=True)\n",
       "  (classifier): Identity()\n",
       ")"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_extractor_model.classifier=torch.nn.Identity()\n",
    "feature_extractor_model=feature_extractor_model.to(device)\n",
    "feature_extractor_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_probab(features, logits=True):\n",
    "    x=np.dot(features,np.transpose(classifier_weights))+classifier_bias\n",
    "    if logits:\n",
    "        return x\n",
    "    e_x = np.exp(x - np.max(x,axis=0))\n",
    "    return e_x / e_x.sum(axis=1)[:,None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Compose(\n",
      "    Resize(size=(224, 224), interpolation=PIL.Image.BILINEAR)\n",
      "    ToTensor()\n",
      "    Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
      ")\n",
      "/home/HDD6TB/datasets/emotions/ABAW/cropped_aligned\n"
     ]
    }
   ],
   "source": [
    "print(test_transforms)\n",
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "#data_dir=os.path.join(DATA_DIR,'cropped_aligned')\n",
    "print(data_dir)\n",
    "img_names=[]\n",
    "X_global_features=[]\n",
    "imgs=[]\n",
    "for filename in tqdm(os.listdir(data_dir)):\n",
    "    frames_dir=os.path.join(data_dir,filename)    \n",
    "    for img_name in os.listdir(frames_dir):\n",
    "        if img_name.lower().endswith('.jpg'):\n",
    "            img = Image.open(os.path.join(frames_dir,img_name))\n",
    "            img_tensor = test_transforms(img)\n",
    "            if img.size:\n",
    "                img_names.append(filename+'/'+img_name)\n",
    "                imgs.append(img_tensor)\n",
    "                if len(imgs)>=96: #48: #64: #32:        \n",
    "                    features = feature_extractor_model(torch.stack(imgs, dim=0).to(device))\n",
    "                    features=features.data.cpu().numpy()\n",
    "                    \n",
    "                    if len(X_global_features)==0:\n",
    "                        X_global_features=features\n",
    "                    else:\n",
    "                        X_global_features=np.concatenate((X_global_features,features),axis=0)\n",
    "                    imgs=[]\n",
    "\n",
    "if len(imgs)>0:        \n",
    "    features = feature_extractor_model(torch.stack(imgs, dim=0).to(device))\n",
    "    features=features.data.cpu().numpy()\n",
    "\n",
    "    if len(X_global_features)==0:\n",
    "        X_global_features=features\n",
    "    else:\n",
    "        X_global_features=np.concatenate((X_global_features,features),axis=0)\n",
    "\n",
    "    imgs=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_scores=get_probab(X_global_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2657113\n"
     ]
    }
   ],
   "source": [
    "filename2featuresAll={img_name:(global_features,scores) for img_name,global_features,scores in zip(img_names,X_global_features,X_scores)}\n",
    "print(len(filename2featuresAll))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save/load features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "enet_b0_8_best_vgaf.pickle\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "if False:\n",
    "    model_name='mymobilenet_7_ft_sgd_model'\n",
    "    num_classes=7\n",
    "else:\n",
    "    num_classes=8\n",
    "    model_name='enet_b0_8_best_vgaf' #first three challenges\n",
    "    #model_name='enet2_8' #MTL challenge\n",
    "    \n",
    "MODEL2FEATURES=model_name+'.pickle' \n",
    "\n",
    "print(MODEL2FEATURES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    with open(MODEL2FEATURES, 'wb') as handle:\n",
    "        pickle.dump(filename2featuresAll, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2658822\n"
     ]
    }
   ],
   "source": [
    "#with open('../../../emotions-multimodal/faces/ABAW/'+MODEL2FEATURES, 'rb') as handle:\n",
    "with open(MODEL2FEATURES, 'rb') as handle:\n",
    "    filename2featuresAll=pickle.load(handle)\n",
    "print(len(filename2featuresAll))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model,Sequential, load_model,model_from_json\n",
    "from tensorflow.keras.applications import mobilenet,mobilenet_v2,densenet,inception_resnet_v2,inception_v3,vgg16,resnet_v2,resnet\n",
    "import efficientnet.tfkeras as enet\n",
    "from tensorflow.keras.layers import Flatten, Dense, Dropout,GlobalAveragePooling2D,Activation, Conv2D, Reshape,DepthwiseConv2D,Input\n",
    "from tensorflow.keras.optimizers import SGD, Adam\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard, Callback, CSVLogger, EarlyStopping\n",
    "from tensorflow.keras.metrics import top_k_categorical_accuracy\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from copy import deepcopy\n",
    "class SaveBestModel(tf.keras.callbacks.Callback):\n",
    "    def __init__(self, save_best_metric='val_loss', this_max=False):\n",
    "        self.save_best_metric = save_best_metric\n",
    "        self.max = this_max\n",
    "        if this_max:\n",
    "            self.best = float('-inf')\n",
    "        else:\n",
    "            self.best = float('inf')\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        metric_value = logs[self.save_best_metric]\n",
    "        if self.max:\n",
    "            if metric_value > self.best:\n",
    "                self.best = metric_value\n",
    "                self.best_model_weights = deepcopy(self.model.get_weights())\n",
    "\n",
    "        else:\n",
    "            if metric_value < self.best:\n",
    "                self.best = metric_value\n",
    "                self.best_model_weights = deepcopy(self.model.get_weights())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_transfer(label,class_num):\n",
    "    return np.eye(class_num)[label]\n",
    "\n",
    "def metric_for_Exp(gt,pred,class_num=8):\n",
    "    acc = accuracy_score(gt,pred)\n",
    "    \n",
    "    # compute_F1\n",
    "    gt = one_hot_transfer(gt,class_num)\n",
    "    pred = one_hot_transfer(pred,class_num)\n",
    "    F1 = []\n",
    "    for i in range(class_num):\n",
    "        gt_ = gt[:,i]\n",
    "        pred_ = pred[:,i]\n",
    "        F1.append(f1_score(gt_.flatten(), pred_))\n",
    "    F1_mean = np.mean(F1)\n",
    "    return F1_mean,acc,F1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CCC_score(x, y):\n",
    "    vx = x - np.mean(x)\n",
    "    vy = y - np.mean(y)\n",
    "    rho = np.sum(vx * vy) / (np.sqrt(np.sum(vx**2)) * np.sqrt(np.sum(vy**2)))\n",
    "    x_m = np.mean(x)\n",
    "    y_m = np.mean(y)\n",
    "    x_s = np.std(x)\n",
    "    y_s = np.std(y)\n",
    "    ccc = 2*rho*x_s*y_s/(x_s**2 + y_s**2 + (x_m - y_m)**2)\n",
    "    return ccc\n",
    "\n",
    "def metric_for_VA(gt_V,gt_A,pred_V,pred_A):\n",
    "    ccc_V,ccc_A = CCC_score(gt_V,pred_V),CCC_score(gt_A,pred_A)\n",
    "    return ccc_V,ccc_A, 0.5*(ccc_V+ccc_A)\n",
    "\n",
    "def CCC_numpy(y_true, y_pred):\n",
    "    '''Reference numpy implementation of Lin's Concordance correlation coefficient'''\n",
    "    \n",
    "    # covariance between y_true and y_pred\n",
    "    s_xy = np.cov([y_true, y_pred])[0,1]\n",
    "    # means\n",
    "    x_m = np.mean(y_true)\n",
    "    y_m = np.mean(y_pred)\n",
    "    # variances\n",
    "    s_x_sq = np.var(y_true)\n",
    "    s_y_sq = np.var(y_pred)\n",
    "    \n",
    "    # condordance correlation coefficient\n",
    "    ccc = (2.0*s_xy) / (s_x_sq + s_y_sq + (x_m-y_m)**2)\n",
    "    \n",
    "    return ccc\n",
    "\n",
    "def CCC(y_true, y_pred):\n",
    "    '''Lin's Concordance correlation coefficient: https://en.wikipedia.org/wiki/Concordance_correlation_coefficient\n",
    "    \n",
    "    The concordance correlation coefficient is the correlation between two variables that fall on the 45 degree line through the origin.\n",
    "    \n",
    "    It is a product of\n",
    "    - precision (Pearson correlation coefficient) and\n",
    "    - accuracy (closeness to 45 degree line)\n",
    "\n",
    "    Interpretation:\n",
    "    - `rho_c =  1` : perfect agreement\n",
    "    - `rho_c =  0` : no agreement\n",
    "    - `rho_c = -1` : perfect disagreement \n",
    "    \n",
    "    Args: \n",
    "    - y_true: ground truth\n",
    "    - y_pred: predicted values\n",
    "    \n",
    "    Returns:\n",
    "    - concordance correlation coefficient (float)\n",
    "    '''\n",
    "    \n",
    "    # covariance between y_true and y_pred\n",
    "    s_xy = K.mean((y_true - K.mean(y_true)) * (y_pred - K.mean(y_pred)))\n",
    "    # means\n",
    "    x_m = K.mean(y_true)\n",
    "    y_m = K.mean(y_pred)\n",
    "    # variances\n",
    "    s_x_sq = K.var(y_true)\n",
    "    s_y_sq = K.var(y_pred)\n",
    "    \n",
    "    # condordance correlation coefficient\n",
    "    ccc = (2.0*s_xy) / (s_x_sq + s_y_sq + (x_m-y_m)**2+K.epsilon())\n",
    "    return ccc\n",
    "\n",
    "def CCC_VA(y_true, y_pred):\n",
    "    return 1-0.5*(CCC(y_true[:,0], y_pred[:,0])+CCC(y_true[:,1], y_pred[:,1]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "def f1_score_max_for_AU_one_class(gt, pred, thresh,type=0):\n",
    "    gt = gt[:,type]\n",
    "    pred = pred[:,type]\n",
    "    P = []\n",
    "    R = []\n",
    "    ACC = []\n",
    "    F1 = []\n",
    "    for i in thresh:\n",
    "        new_pred = ((pred >= i) * 1).flatten()\n",
    "        P.append(precision_score(gt.flatten(), new_pred))\n",
    "        R.append(recall_score(gt.flatten(), new_pred))\n",
    "        ACC.append(accuracy_score(gt.flatten(), new_pred))\n",
    "        F1.append(f1_score(gt.flatten(), new_pred))\n",
    "\n",
    "    F1_MAX = max(F1)\n",
    "    if F1_MAX < 0 or math.isnan(F1_MAX):\n",
    "        F1_MAX = 0\n",
    "        F1_THRESH = 0\n",
    "        accuracy = 0\n",
    "    else:\n",
    "        idx_thresh = np.argmax(F1)\n",
    "        F1_THRESH = thresh[idx_thresh]\n",
    "        accuracy = ACC[idx_thresh]\n",
    "    return F1,F1_MAX,F1_THRESH,accuracy\n",
    "\n",
    "def f1_score_max(gt, pred, thresh,c=12):\n",
    "    F1_s = []\n",
    "    F1_t = []\n",
    "    ACC = []\n",
    "    from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score\n",
    "    for i in range(c):\n",
    "        F1, F1_MAX, F1_THRESH,acc = f1_score_max_for_AU_one_class(gt,pred,thresh,i)\n",
    "        F1_s.append(F1_MAX)\n",
    "        F1_t.append(F1_THRESH)\n",
    "        ACC.append(acc)\n",
    "    F1_s=np.array(F1_s)\n",
    "    F1_t=np.array(F1_t)\n",
    "    ACC=np.array(ACC)\n",
    "    return F1_s.mean(),F1_t.mean(),F1_s,F1_t,ACC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Expr "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(585317, 1280) (585317,) 12044 1\n",
      "(280532, 1280) (280532,) 3698 3\n"
     ]
    }
   ],
   "source": [
    "def get_image2Expr(dirname):\n",
    "    dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/EXPR_Classification_Challenge/',dirname)\n",
    "    num_missed=0\n",
    "    X,y=[],[]\n",
    "    minConstantFrames,numConstant=100000,0\n",
    "    for filename in os.listdir(dirpath):\n",
    "        fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "        if ext.lower()=='.txt':\n",
    "            with open(os.path.join(dirpath,filename)) as f:\n",
    "                lines = f.read().splitlines()\n",
    "                prev_val=None\n",
    "                for i,line in enumerate(lines):\n",
    "                    if i>0:\n",
    "                        expression=int(line)\n",
    "                        if expression>=0:\n",
    "                            if prev_val is None:\n",
    "                                prev_val=expression\n",
    "                                numConstant=1\n",
    "                            elif prev_val==expression:\n",
    "                                numConstant+=1\n",
    "                            else:\n",
    "                                if numConstant<minConstantFrames:\n",
    "                                    minConstantFrames=numConstant\n",
    "                                prev_val=expression\n",
    "                                numConstant=1\n",
    "                            imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                            if imagename in filename2featuresAll:\n",
    "                                X.append(filename2featuresAll[imagename][0])\n",
    "                                y.append(expression)\n",
    "                            else:\n",
    "                                num_missed+=1\n",
    "    X=np.array(X)\n",
    "    y=np.array(y)\n",
    "    print(X.shape,y.shape,num_missed,minConstantFrames)\n",
    "    return X,y\n",
    "\n",
    "X_train,y_train=get_image2Expr('Train_Set')\n",
    "X_val,y_val=get_image2Expr('Validation_Set')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_expr():\n",
    "    y_val_preds=mlpModel.predict(X_val)\n",
    "    y_pred=np.argmax(y_val_preds,axis=1)\n",
    "    print('Acc:',(y_pred==y_val).mean(), 'F1:',f1_score(y_true=y_val,y_pred=y_pred, average=\"macro\"))\n",
    "    print(metric_for_Exp(y_val,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[177198  16573  10771   9080  95463  78751  31615 165866] {0: 1.0, 1: 10.691968865021419, 2: 16.451397270448425, 3: 19.515198237885464, 4: 1.856195594104522, 5: 2.2501047605744686, 6: 5.6048711054879, 7: 1.0683202102902343} 8 [0 1 2 3 4 5 6 7]\n"
     ]
    }
   ],
   "source": [
    "(unique, counts) = np.unique(y_train, return_counts=True)\n",
    "num_classes=len(unique)\n",
    "cw=1/counts\n",
    "cw/=cw.min()\n",
    "class_weights = {i:cwi for i,cwi in zip(unique,cw)}\n",
    "print(counts, class_weights, num_classes, unique)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=256 #128\n",
    "mlpModel=Sequential()\n",
    "if False:\n",
    "    mlpModel.add(Dense(num_classes, input_shape=X_train.shape[1:],activation='softmax',use_bias=True,kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size)))\n",
    "else:\n",
    "    mlpModel.add(Dense(128, input_shape=X_train.shape[1:],activation='relu')) #256\n",
    "    mlpModel.add(Dense(num_classes,activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 128)               163968    \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 8)                 1032      \n",
      "=================================================================\n",
      "Total params: 165,000\n",
      "Trainable params: 165,000\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "2287/2287 [==============================] - 8s 4ms/step - loss: 1.5398 - accuracy: 0.7604 - val_loss: 1.9589 - val_accuracy: 0.4541\n",
      "Epoch 2/10\n",
      "2287/2287 [==============================] - 7s 3ms/step - loss: 0.5820 - accuracy: 0.8989 - val_loss: 2.2295 - val_accuracy: 0.4642\n",
      "Epoch 3/10\n",
      "2287/2287 [==============================] - 8s 3ms/step - loss: 0.4193 - accuracy: 0.9240 - val_loss: 2.5471 - val_accuracy: 0.4654\n",
      "Epoch 4/10\n",
      "2287/2287 [==============================] - 8s 3ms/step - loss: 0.3415 - accuracy: 0.9366 - val_loss: 2.7357 - val_accuracy: 0.4519\n",
      "Epoch 5/10\n",
      "2287/2287 [==============================] - 7s 3ms/step - loss: 0.2895 - accuracy: 0.9458 - val_loss: 2.8545 - val_accuracy: 0.4382\n",
      "Epoch 6/10\n",
      "2287/2287 [==============================] - 7s 3ms/step - loss: 0.2478 - accuracy: 0.9526 - val_loss: 3.0922 - val_accuracy: 0.4332\n",
      "Epoch 7/10\n",
      "2287/2287 [==============================] - 7s 3ms/step - loss: 0.2157 - accuracy: 0.9575 - val_loss: 3.2851 - val_accuracy: 0.4280\n",
      "Epoch 8/10\n",
      "2287/2287 [==============================] - 7s 3ms/step - loss: 0.1958 - accuracy: 0.9613 - val_loss: 3.3189 - val_accuracy: 0.4466\n",
      "Epoch 9/10\n",
      "2287/2287 [==============================] - 8s 3ms/step - loss: 0.1779 - accuracy: 0.9654 - val_loss: 3.5852 - val_accuracy: 0.4007\n",
      "Epoch 10/10\n",
      "2287/2287 [==============================] - 8s 3ms/step - loss: 0.1909 - accuracy: 0.9651 - val_loss: 3.7639 - val_accuracy: 0.4298\n",
      "0.4654121398925781\n"
     ]
    }
   ],
   "source": [
    "mlpModel.compile(optimizer=Adam(lr=1e-3), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "mlpModel.summary()\n",
    "\n",
    "save_best_model = SaveBestModel('val_accuracy',True)\n",
    "mlpModel.fit(X_train,y_train, batch_size=batch_size, epochs=10, verbose=1, \n",
    "             callbacks=[save_best_model], validation_data=(X_val,y_val),class_weight=class_weights)\n",
    "best_model_weights = save_best_model.best_model_weights\n",
    "print(save_best_model.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 0.4298012347967433 F1: 0.3418451036131609\n",
      "(0.3418451036131609, 0.4298012347967433, [0.5287005218276696, 0.18212996389891697, 0.5914928499863376, 0.018604651162790694, 0.46076099313138263, 0.3311925510325892, 0.22376455670859888, 0.39811474115700163])\n",
      "Best weights:\n",
      "Acc: 0.4654121454949881 F1: 0.34060359780179217\n",
      "(0.34060359780179217, 0.4654121454949881, [0.5619842406043042, 0.1331245105716523, 0.4915926179084074, 0.009731543624161074, 0.4858991788569254, 0.3211159481346253, 0.23595084924606013, 0.4854298934682019])\n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "    print_expr()\n",
    "    print('Best weights:')\n",
    "    mlpModel.set_weights(best_model_weights)\n",
    "print_expr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Acc: 0.500285172458044 F1: 0.3807067519117502\n",
      "(0.3807067519117502, 0.500285172458044, [0.608643679809698, 0.15128569174948728, 0.5162256615077384, 0.01599360255897641, 0.4778312968664514, 0.4611138100075078, 0.3029224246739377, 0.5116378481202044])\n"
     ]
    }
   ],
   "source": [
    "if False:\n",
    "    #mlpModel.save_weights('expr_enet0_7.h5') #Acc: 0.48438324326636534 F1: 0.3604978104476117\n",
    "    mlpModel.save_weights('expr_enet0_vgaf.h5')#Acc: 0.500285172458044 F1: 0.3807067519117502\n",
    "else:\n",
    "    #mlpModel.load_weights('../../../emotions-multimodal/faces/ABAW/expr_enet0_vgaf.h5')\n",
    "    mlpModel.load_weights('expr_enet0_vgaf.h5')\n",
    "    print_expr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Smooth validation predictions  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 70/70 [00:22<00:00,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/EXPR_Classification_Challenge/Validation_Set')\n",
    "test_videos={}\n",
    "for filename in tqdm(os.listdir(dirpath)):\n",
    "    fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "    if ext.lower()=='.txt':\n",
    "        X,indices,expressions=[],[],[]\n",
    "        with open(os.path.join(dirpath,filename)) as f:\n",
    "            lines = f.read().splitlines()\n",
    "            prev_val=None\n",
    "            for i,line in enumerate(lines):\n",
    "                if i>0:\n",
    "                    imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                    if imagename in filename2featuresAll:\n",
    "                        X.append(filename2featuresAll[imagename][0])\n",
    "                        indices.append(i)\n",
    "                        expressions.append(int(line))\n",
    "        test_videos[fn]=(mlpModel.predict(np.array(X)),indices,np.array(expressions))\n",
    "print(len(test_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean 0 Acc: 0.500285172458044 F1: 0.3807067519117502\n",
      "mean 2 Acc: 0.5134173641509703 F1: 0.3914238704567333\n",
      "median 2 Acc: 0.5104373119644104 F1: 0.38878476888647095\n",
      "mean 7 Acc: 0.5263855816805213 F1: 0.4018379088214337\n",
      "median 7 Acc: 0.5236907019520055 F1: 0.39958196647074146\n"
     ]
    }
   ],
   "source": [
    "hyperparams=[(isMean,delta) for delta in [0,2,7]  for isMean in [1,0] if not (isMean==0 and delta==0)]\n",
    "total_true=[]\n",
    "total_preds=[[] for _ in range(len(hyperparams))]\n",
    "for videoname,(y_pred_expr,indices,expressions) in test_videos.items():\n",
    "    for i,ind in enumerate(indices):\n",
    "        if expressions[i]>=0:\n",
    "            total_true.append(expressions[i])\n",
    "    cur_ind=0\n",
    "    preds_proba=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds_proba.append(y_pred_expr[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds_proba.append(y_pred_expr[cur_ind])\n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_expr[cur_ind-1]+(1-w)*y_pred_expr[cur_ind]\n",
    "                preds_proba.append(pred)\n",
    "    \n",
    "    preds_proba=np.array(preds_proba)\n",
    "    for hInd,(isMean,delta) in enumerate(hyperparams):\n",
    "        preds=[]\n",
    "        for i in range(len(preds_proba)):\n",
    "            i1=max(i-delta,0)\n",
    "            if isMean:\n",
    "                proba=np.mean(preds_proba[i1:i+delta+1],axis=0)\n",
    "            else:\n",
    "                proba=np.median(preds_proba[i1:i+delta+1],axis=0)\n",
    "            preds.append(np.argmax(proba))\n",
    "        for i,ind in enumerate(indices):\n",
    "            if expressions[i]>=0:\n",
    "                total_preds[hInd].append(preds[ind-1])\n",
    "\n",
    "total_true=np.array(total_true)\n",
    "for hInd,(isMean,delta) in enumerate(hyperparams):\n",
    "    preds=np.array(total_preds[hInd])\n",
    "    print('mean' if isMean else 'median',delta,'Acc:',(preds==total_true).mean(), 'F1:',f1_score(y_true=total_true,y_pred=preds, average=\"macro\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adaptive Frame Rate "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "source code for the \"Facial Expression Recognition with Adaptive Frame Rate based on Multiple Testing Correction\" accepted at ICML 2023"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Acc: 0.500285172458044 F1: 0.3807067519117502\n",
      "5 Acc: 0.5220830422197824 F1: 0.39793975387287783\n",
      "15 Acc: 0.5355859581081659 F1: 0.40971928710644606\n",
      "25 Acc: 0.5427259635264426 F1: 0.4173622193839083\n",
      "50 Acc: 0.5499301327477792 F1: 0.4264244745156282\n",
      "100 Acc: 0.5512134088089772 F1: 0.42621218115321136\n",
      "200 Acc: 0.5428614204440135 F1: 0.4230475710116932\n",
      "500 Acc: 0.5309911168779319 F1: 0.4218031114287871\n"
     ]
    }
   ],
   "source": [
    "deltas=[0,5,15,25,50,100,200,500]\n",
    "total_true=[]\n",
    "total_preds=[[] for _ in range(len(deltas))]\n",
    "for videoname,(y_pred_expr,indices,expressions) in test_videos.items():\n",
    "    for i,ind in enumerate(indices):\n",
    "        if expressions[i]>=0:\n",
    "            total_true.append(expressions[i])\n",
    "    cur_ind=0\n",
    "    preds_proba=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds_proba.append(y_pred_expr[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds_proba.append(y_pred_expr[cur_ind])\n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_expr[cur_ind-1]+(1-w)*y_pred_expr[cur_ind]\n",
    "                preds_proba.append(pred)\n",
    "    \n",
    "    preds_proba=np.array(preds_proba)\n",
    "    for hInd,delta in enumerate(deltas):\n",
    "        preds=[]\n",
    "        for i in range(len(preds_proba)):\n",
    "            i1=max(i-delta,0)\n",
    "            proba=np.mean(preds_proba[i1:i+delta+1],axis=0)\n",
    "            preds.append(np.argmax(proba))\n",
    "        for i,ind in enumerate(indices):\n",
    "            if expressions[i]>=0:\n",
    "                total_preds[hInd].append(preds[ind-1])\n",
    "\n",
    "total_true=np.array(total_true)\n",
    "for hInd,delta in enumerate(deltas):\n",
    "    preds=np.array(total_preds[hInd])\n",
    "    print(delta,'Acc:',(preds==total_true).mean(), 'F1:',f1_score(y_true=total_true,y_pred=preds, average=\"macro\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta=100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 248/248 [00:56<00:00,  4.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "248\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/EXPR_Classification_Challenge/Train_Set')\n",
    "train_videos={}\n",
    "for filename in tqdm(os.listdir(dirpath)):\n",
    "    fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "    if ext.lower()=='.txt':\n",
    "        X,indices,expressions=[],[],[]\n",
    "        with open(os.path.join(dirpath,filename)) as f:\n",
    "            lines = f.read().splitlines()\n",
    "            prev_val=None\n",
    "            for i,line in enumerate(lines):\n",
    "                if i>0:\n",
    "                    imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                    if imagename in filename2featuresAll:\n",
    "                        X.append(filename2featuresAll[imagename][0])\n",
    "                        indices.append(i)\n",
    "                        expressions.append(int(line))\n",
    "        train_videos[fn]=(mlpModel.predict(np.array(X)),indices,np.array(expressions))\n",
    "print(len(train_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stride 200\n",
      "stride 100\n",
      "stride 50\n",
      "stride 25\n",
      "stride 10\n"
     ]
    }
   ],
   "source": [
    "stride2scores={}\n",
    "for stride in [200,100,50,25,10]:\n",
    "    print('stride',stride)\n",
    "    y_total_train,predictions,max_decision_values=[],[],[]\n",
    "    for (y_pred_expr,indices,expressions) in train_videos.values():\n",
    "        emotional_indices=[]\n",
    "        for i,ind in enumerate(indices):\n",
    "            if expressions[i]>=0:\n",
    "                y_total_train.append(expressions[i])\n",
    "                emotional_indices.append(ind-1)\n",
    "        cur_ind=0\n",
    "        preds_proba=[]\n",
    "        for i in range(indices[-1]):\n",
    "            if indices[cur_ind]-1==i:\n",
    "                preds_proba.append(y_pred_expr[cur_ind])\n",
    "                cur_ind+=1\n",
    "            else:\n",
    "                if cur_ind==0:\n",
    "                    preds_proba.append(y_pred_expr[cur_ind])\n",
    "                else:\n",
    "                    w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                    pred=w*y_pred_expr[cur_ind-1]+(1-w)*y_pred_expr[cur_ind]\n",
    "                    preds_proba.append(pred)\n",
    "\n",
    "        preds_proba=np.array(preds_proba)\n",
    "        for i in range(len(emotional_indices)):\n",
    "            i1=max(emotional_indices[i]-delta,0)\n",
    "            cur_preds=preds_proba[i1:emotional_indices[i]+delta+1:stride]\n",
    "            proba=np.mean(cur_preds,axis=0)\n",
    "            best_ind=np.argmax(proba)\n",
    "            predictions.append(best_ind)\n",
    "            max_decision_values.append(proba[best_ind])\n",
    "\n",
    "    stride2scores[stride]=(np.array(y_total_train),np.array(predictions),np.array(max_decision_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200 best_threshold 0.68035233 412187\n",
      "100 best_threshold 0.543008 494777\n",
      "50 best_threshold 0.5162231 508738\n",
      "25 best_threshold 0.4995789 518070\n",
      "10 best_threshold 0.49043015 523199\n",
      "{200: 0.68035233, 100: 0.543008, 50: 0.5162231, 25: 0.4995789, 10: 0.49043015, 1: 0}\n"
     ]
    }
   ],
   "source": [
    "def get_threshold(stride,fpr_corrected):\n",
    "    (y_total_train,predictions,max_decision_values)=stride2scores[stride]\n",
    "    mistakes=max_decision_values[predictions!=y_total_train]\n",
    "    best_threshold=-1\n",
    "    for i,threshold in enumerate(sorted(max_decision_values[predictions==y_total_train])[::-1]):\n",
    "        tpr=i/len(predictions)\n",
    "        fpr=(mistakes>threshold).sum()/len(predictions)\n",
    "        #print(threshold,fpr,tpr)\n",
    "        if fpr>fpr_corrected:\n",
    "            if best_threshold==-1:\n",
    "                best_threshold=threshold\n",
    "            print(stride,'best_threshold',best_threshold,i)\n",
    "            break\n",
    "        best_threshold=threshold\n",
    "    return best_threshold\n",
    "\n",
    "stride2threshold={}\n",
    "for stride in stride2scores:\n",
    "    fpr_corrected=0.05\n",
    "    stride2threshold[stride]=get_threshold(stride,fpr_corrected)\n",
    "stride2threshold[1]=0\n",
    "print(stride2threshold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200, 100, 50, 10, 1]\n",
      "Acc: 0.5402164458956554 F1: 0.41902184233325\n",
      "11262799 55906719 0.2014569840880843\n",
      "[50, 25, 1]\n",
      "Acc: 0.549876662911896 F1: 0.42571232560361677\n",
      "14860715 55906719 0.2658126834450793\n",
      "[50, 10, 1]\n",
      "Acc: 0.549947956026407 F1: 0.42578430647859183\n",
      "13992904 55906719 0.25029020214904757\n",
      "[200, 50, 1]\n",
      "Acc: 0.5424015798554176 F1: 0.42027871842106457\n",
      "16430120 55906719 0.29388453291276134\n",
      "[100, 50, 1]\n",
      "Acc: 0.5455919467297848 F1: 0.42246531152184047\n",
      "15241987 55906719 0.2726324719574404\n",
      "[200, 1]\n",
      "Acc: 0.5432428386066473 F1: 0.42046923182790485\n",
      "32631266 55906719 0.583673422151638\n",
      "[100, 1]\n",
      "Acc: 0.5461587269901473 F1: 0.42277716409351745\n",
      "20393603 55906719 0.36477910642547273\n",
      "[50, 1]\n",
      "Acc: 0.5500085551737414 F1: 0.42583630334743344\n",
      "18456371 55906719 0.3301279583228628\n",
      "[200]\n",
      "Acc: 0.4808471047866197 F1: 0.36236023497146447\n",
      "551838 55906719 0.009870691928818074\n",
      "[100]\n",
      "Acc: 0.5232558139534884 F1: 0.40446631863369187\n",
      "832370 55906719 0.014888550336856649\n",
      "[50]\n",
      "Acc: 0.537835255870988 F1: 0.41651093902176317\n",
      "1388638 55906719 0.02483848139970439\n",
      "[25]\n",
      "Acc: 0.5469857271184749 F1: 0.4253450173566402\n",
      "2501186 55906719 0.04473855816865232\n",
      "[10]\n",
      "Acc: 0.5492742360942781 F1: 0.42477224067855746\n",
      "5839006 55906719 0.10444193657653206\n",
      "[1]\n",
      "Acc: 0.5512134088089772 F1: 0.42621218115321136\n",
      "55906719 55906719 1.0\n"
     ]
    }
   ],
   "source": [
    "all_strides=[\n",
    "    [200, 100, 50, 10, 1],\n",
    "    [50, 25, 1],\n",
    "    [50, 10, 1],\n",
    "    [200,50,1],\n",
    "    [100,50,1],\n",
    "    [200,1],\n",
    "    [100,1],\n",
    "    [50,1]\n",
    "]\n",
    "for s in stride2threshold.keys():\n",
    "    all_strides.append([s])\n",
    "\n",
    "for strides in all_strides:\n",
    "    print(strides)\n",
    "    last_stride=strides[-1]\n",
    "\n",
    "    total_true=[]\n",
    "    total_preds=[]\n",
    "    total_frames_processed,total_frames=0,0\n",
    "    for (y_pred_expr,indices,expressions) in test_videos.values():\n",
    "        emotional_indices=[]\n",
    "        for i,ind in enumerate(indices):\n",
    "            if expressions[i]>=0:\n",
    "                total_true.append(expressions[i])\n",
    "                emotional_indices.append(ind-1)\n",
    "        cur_ind=0\n",
    "        preds_proba=[]\n",
    "        for i in range(indices[-1]):\n",
    "            if indices[cur_ind]-1==i:\n",
    "                preds_proba.append(y_pred_expr[cur_ind])\n",
    "                cur_ind+=1\n",
    "            else:\n",
    "                if cur_ind==0:\n",
    "                    preds_proba.append(y_pred_expr[cur_ind])\n",
    "                else:\n",
    "                    w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                    pred=w*y_pred_expr[cur_ind-1]+(1-w)*y_pred_expr[cur_ind]\n",
    "                    preds_proba.append(pred)\n",
    "\n",
    "        preds_proba=np.array(preds_proba)\n",
    "\n",
    "        preds=-np.ones(len(emotional_indices))\n",
    "        for stride in strides:\n",
    "            threshold=stride2threshold[stride]\n",
    "            for i in range(len(emotional_indices)):\n",
    "                if preds[i]<0:\n",
    "                    i1=max(emotional_indices[i]-delta,0)\n",
    "                    cur_preds=preds_proba[i1:emotional_indices[i]+delta+1:stride]\n",
    "                    proba=np.mean(cur_preds,axis=0)\n",
    "                    best_ind=np.argmax(proba)\n",
    "                    if proba[best_ind]>=threshold or stride==last_stride:\n",
    "                        total_frames_processed+=len(cur_preds)\n",
    "                        total_frames+=len(preds_proba[i1:emotional_indices[i]+delta+1])\n",
    "                        preds[i]=best_ind\n",
    "        for p in preds:\n",
    "            total_preds.append(p)\n",
    "\n",
    "    total_true=np.array(total_true)\n",
    "    preds=np.array(total_preds)\n",
    "    print('Acc:',(preds==total_true).mean(), 'F1:',f1_score(y_true=total_true,y_pred=preds, average=\"macro\"))\n",
    "    print(total_frames_processed,total_frames,total_frames_processed/total_frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200, 100, 50, 10, 1]\n",
      "stride 200 0.02\n",
      "200 best_threshold 0.8820417 341051\n",
      "stride 100 0.04\n",
      "100 best_threshold 0.58247334 482013\n",
      "stride 50 0.06000000000000001\n",
      "50 best_threshold 0.4818677 517646\n",
      "stride 10 0.08\n",
      "Acc: 0.5439914163090128 F1: 0.42174291492447946\n",
      "2069089 55906719 0.03700966604747454\n",
      "[50, 25, 1]\n",
      "stride 50 0.03333333333333333\n",
      "50 best_threshold 0.5796947 487628\n",
      "stride 25 0.06666666666666667\n",
      "25 best_threshold 0.41966972 532157\n",
      "Acc: 0.5496378309782841 F1: 0.4252566581975647\n",
      "9573269 55906719 0.17123646622868352\n",
      "[50, 10, 1]\n",
      "stride 50 0.03333333333333333\n",
      "50 best_threshold 0.5796947 487628\n",
      "stride 10 0.06666666666666667\n",
      "10 best_threshold 0.39557225 537192\n",
      "Acc: 0.5503400681562175 F1: 0.4257837197103246\n",
      "8673566 55906719 0.15514353471538905\n",
      "[200, 50, 1]\n",
      "stride 200 0.03333333333333333\n",
      "200 best_threshold 0.78248155 382474\n",
      "stride 50 0.06666666666666667\n",
      "50 best_threshold 0.44965786 523290\n",
      "Acc: 0.5435458343433192 F1: 0.4207483426245095\n",
      "11408607 55906719 0.20406504270086034\n",
      "[100, 50, 1]\n",
      "stride 100 0.03333333333333333\n",
      "100 best_threshold 0.61147404 471496\n",
      "stride 50 0.06666666666666667\n",
      "50 best_threshold 0.44965786 523290\n",
      "Acc: 0.5459377183351631 F1: 0.42295849428037174\n",
      "11356067 55906719 0.20312526299388092\n",
      "[200, 1]\n",
      "stride 200 0.05\n",
      "200 best_threshold 0.68035233 412187\n",
      "Acc: 0.5432428386066473 F1: 0.42046923182790485\n",
      "32631266 55906719 0.583673422151638\n",
      "[100, 1]\n",
      "stride 100 0.05\n",
      "100 best_threshold 0.543008 494777\n",
      "Acc: 0.5461587269901473 F1: 0.42277716409351745\n",
      "20393603 55906719 0.36477910642547273\n",
      "[50, 1]\n",
      "stride 50 0.05\n",
      "50 best_threshold 0.5162231 508738\n",
      "Acc: 0.5500085551737414 F1: 0.42583630334743344\n",
      "18456371 55906719 0.3301279583228628\n",
      "[200]\n",
      "Acc: 0.4808471047866197 F1: 0.36236023497146447\n",
      "551838 55906719 0.009870691928818074\n",
      "[100]\n",
      "Acc: 0.5232558139534884 F1: 0.40446631863369187\n",
      "832370 55906719 0.014888550336856649\n",
      "[50]\n",
      "Acc: 0.537835255870988 F1: 0.41651093902176317\n",
      "1388638 55906719 0.02483848139970439\n",
      "[25]\n",
      "Acc: 0.5469857271184749 F1: 0.4253450173566402\n",
      "2501186 55906719 0.04473855816865232\n",
      "[10]\n",
      "Acc: 0.5492742360942781 F1: 0.42477224067855746\n",
      "5839006 55906719 0.10444193657653206\n",
      "[1]\n",
      "Acc: 0.5512134088089772 F1: 0.42621218115321136\n",
      "55906719 55906719 1.0\n"
     ]
    }
   ],
   "source": [
    "#Complete example\n",
    "all_strides=[\n",
    "    [200, 100, 50, 10, 1],\n",
    "    [50, 25, 1],\n",
    "    [50, 10, 1],\n",
    "    [200,50,1],\n",
    "    [100,50,1],\n",
    "    [200,1],\n",
    "    [100,1],\n",
    "    [50,1]\n",
    "]\n",
    "for s in stride2threshold.keys():\n",
    "    all_strides.append([s])\n",
    "\n",
    "#all_strides=[[200,1]]\n",
    "for strides in all_strides:\n",
    "    print(strides)\n",
    "    last_stride=strides[-1]\n",
    "    stride2threshold_cur={}\n",
    "    for s_ind,stride in enumerate(strides):\n",
    "        if stride!=last_stride:\n",
    "            fpr_corrected=0.1*(s_ind+1)/(len(strides)) #0.08 #0.1 #\n",
    "            print('stride',stride,fpr_corrected)\n",
    "            stride2threshold_cur[stride]=get_threshold(stride,fpr_corrected)\n",
    "        else:\n",
    "            stride2threshold_cur[stride]=0\n",
    "\n",
    "    total_true=[]\n",
    "    total_preds=[]\n",
    "    total_frames_processed,total_frames=0,0\n",
    "    for (y_pred_expr,indices,expressions) in test_videos.values():\n",
    "        emotional_indices=[]\n",
    "        for i,ind in enumerate(indices):\n",
    "            if expressions[i]>=0:\n",
    "                total_true.append(expressions[i])\n",
    "                emotional_indices.append(ind-1)\n",
    "        cur_ind=0\n",
    "        preds_proba=[]\n",
    "        for i in range(indices[-1]):\n",
    "            if indices[cur_ind]-1==i:\n",
    "                preds_proba.append(y_pred_expr[cur_ind])\n",
    "                cur_ind+=1\n",
    "            else:\n",
    "                if cur_ind==0:\n",
    "                    preds_proba.append(y_pred_expr[cur_ind])\n",
    "                else:\n",
    "                    w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                    pred=w*y_pred_expr[cur_ind-1]+(1-w)*y_pred_expr[cur_ind]\n",
    "                    preds_proba.append(pred)\n",
    "\n",
    "        preds_proba=np.array(preds_proba)\n",
    "\n",
    "        preds=-np.ones(len(emotional_indices))\n",
    "        for stride in strides:\n",
    "            threshold=stride2threshold_cur[stride]\n",
    "            for i in range(len(emotional_indices)):\n",
    "                if preds[i]<0:\n",
    "                    i1=max(emotional_indices[i]-delta,0)\n",
    "                    cur_preds=preds_proba[i1:emotional_indices[i]+delta+1:stride]\n",
    "                    proba=np.mean(cur_preds,axis=0)\n",
    "                    best_ind=np.argmax(proba)\n",
    "                    if proba[best_ind]>=threshold or stride==last_stride:\n",
    "                        total_frames_processed+=len(cur_preds)\n",
    "                        total_frames+=len(preds_proba[i1:emotional_indices[i]+delta+1])\n",
    "                        preds[i]=best_ind\n",
    "        for p in preds:\n",
    "            total_preds.append(p)\n",
    "\n",
    "    total_true=np.array(total_true)\n",
    "    preds=np.array(total_preds)\n",
    "    print('Acc:',(preds==total_true).mean(), 'F1:',f1_score(y_true=total_true,y_pred=preds, average=\"macro\"))\n",
    "    print(total_frames_processed,total_frames,total_frames_processed/total_frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save test results "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228 ['14-30-1920x1080', '16-30-1920x1080', '40-30-1280x720', '43-30-406x720', '44-25-426x240']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(DATA_DIR,'Expression_Classification_Challenge_test_set_release.txt'),'r') as f:\n",
    "    test_set_videos=f.read().splitlines()\n",
    "print(len(test_set_videos),test_set_videos[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 564/564 [00:55<00:00, 10.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "test_videos={}\n",
    "for videoname in tqdm(os.listdir(data_dir)):\n",
    "    if videoname in test_set_videos:\n",
    "        X,indices,filenames=[],[],[]\n",
    "        images=[img_name for img_name in os.listdir(os.path.join(data_dir,videoname)) if img_name.lower().endswith('.jpg')]\n",
    "        for img_name in sorted(images, key=compare_filenames):\n",
    "            k=videoname+'/'+img_name\n",
    "            if k in filename2featuresAll:\n",
    "                X.append(filename2featuresAll[k][0])\n",
    "                indices.append(int(img_name[:-4]))\n",
    "                filenames.append(k)\n",
    "        test_videos[videoname]=(mlpModel.predict(np.array(X)),indices,filenames)\n",
    "print(len(test_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 228/228 [00:08<00:00, 27.56it/s]\n"
     ]
    }
   ],
   "source": [
    "delta=0 # 0 2 4 7\n",
    "resdir=os.path.join(DATA_DIR,'test_results/EXPR/1_single') #2_mean5 5_med15\n",
    "if not os.path.exists(resdir):\n",
    "    os.makedirs(resdir)\n",
    "header = 'Neutral,Anger,Disgust,Fear,Happiness,Sadness,Surprise,Other\\n'\n",
    "for videoname,(y_pred_expr,indices,filenames) in tqdm(test_videos.items()):\n",
    "    cur_ind=0\n",
    "    preds_proba=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds_proba.append(y_pred_expr[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds_proba.append(y_pred_expr[cur_ind])\n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_expr[cur_ind-1]+(1-w)*y_pred_expr[cur_ind]\n",
    "                preds_proba.append(pred)\n",
    "    \n",
    "    pred=y_pred_expr[cur_ind-1]\n",
    "    vn=videoname\n",
    "    if videoname.endswith('_left'):\n",
    "        vn=videoname[:-5]\n",
    "    elif videoname.endswith('_right'):\n",
    "        vn=videoname[:-6]\n",
    "    for _ in range(indices[-1],video2len[vn]):\n",
    "        preds_proba.append(pred)\n",
    "\n",
    "    preds_proba=np.array(preds_proba)\n",
    "    with open(os.path.join(resdir,videoname+'.txt'), 'w') as f:\n",
    "        f.write(header)\n",
    "        for i in range(len(preds_proba)):\n",
    "            i1=max(i-delta,0)\n",
    "            proba=np.mean(preds_proba[i1:i+delta+1],axis=0)\n",
    "            #proba=np.median(preds_proba[i1:i+delta+1],axis=0)\n",
    "            f.write(str(np.argmax(proba))+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Valence-Arousal "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1555919, 8) (1555919, 2) 25265\n",
      "(338755, 8) (338755, 2) 5959\n"
     ]
    }
   ],
   "source": [
    "def get_image2VA(dirname):\n",
    "    dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/VA_Estimation_Challenge/',dirname)\n",
    "    num_missed=0\n",
    "    X,y=[],[]\n",
    "    for filename in os.listdir(dirpath):\n",
    "        fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "        if ext.lower()=='.txt':\n",
    "            with open(os.path.join(dirpath,filename)) as f:\n",
    "                lines = f.read().splitlines()\n",
    "                for i,line in enumerate(lines):\n",
    "                    if i>0:\n",
    "                        splitted_line=line.split(',')\n",
    "                        valence=float(splitted_line[0])\n",
    "                        arousal=float(splitted_line[1])\n",
    "                        if valence>=-1 and arousal>=-1:\n",
    "                            imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                            if imagename in filename2featuresAll:\n",
    "                                X.append(filename2featuresAll[imagename][1])\n",
    "                                y.append((valence,arousal))\n",
    "                            else:\n",
    "                                num_missed+=1\n",
    "    X=np.array(X)\n",
    "    y=np.array(y)\n",
    "    print(X.shape,y.shape,num_missed)\n",
    "    return X,y\n",
    "\n",
    "X_train,y_train=get_image2VA('Train_Set')\n",
    "X_val,y_val=get_image2VA('Validation_Set')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_ccc():\n",
    "    y_val_preds=mlpModel.predict(X_val)\n",
    "    gt_V=y_val[:,0]\n",
    "    gt_A=y_val[:,1]\n",
    "    pred_V=y_val_preds[:,0]\n",
    "    pred_A=y_val_preds[:,1]\n",
    "    print(metric_for_VA(gt_V,gt_A,pred_V,pred_A))\n",
    "    print(CCC_numpy(gt_V,pred_V),CCC_numpy(gt_A,pred_A))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=512 #128\n",
    "mlpModel=Sequential()\n",
    "if True:\n",
    "    mlpModel.add(Dense(2, input_shape=X_train.shape[1:],activation='tanh',use_bias=True,kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size)))\n",
    "else:\n",
    "    mlpModel.add(Dense(128, input_shape=X_train.shape[1:],activation='relu')) #256\n",
    "    mlpModel.add(Dense(2,activation='tanh'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_3 (Dense)              (None, 2)                 18        \n",
      "=================================================================\n",
      "Total params: 18\n",
      "Trainable params: 18\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/20\n",
      "3039/3039 [==============================] - 12s 4ms/step - loss: 0.6652 - mae: 0.4770 - mse: 0.3555 - val_loss: 0.8230 - val_mae: 0.2781 - val_mse: 0.1300\n",
      "Epoch 2/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5047 - mae: 0.2775 - mse: 0.1276 - val_loss: 0.8233 - val_mae: 0.2788 - val_mse: 0.1306\n",
      "Epoch 3/20\n",
      "3039/3039 [==============================] - 10s 3ms/step - loss: 0.5056 - mae: 0.2774 - mse: 0.1276 - val_loss: 0.8243 - val_mae: 0.2821 - val_mse: 0.1330\n",
      "Epoch 4/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5057 - mae: 0.2778 - mse: 0.1279 - val_loss: 0.8234 - val_mae: 0.2775 - val_mse: 0.1294\n",
      "Epoch 5/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5053 - mae: 0.2774 - mse: 0.1276 - val_loss: 0.8243 - val_mae: 0.2811 - val_mse: 0.1324\n",
      "Epoch 6/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5055 - mae: 0.2778 - mse: 0.1279 - val_loss: 0.8210 - val_mae: 0.2746 - val_mse: 0.1275\n",
      "Epoch 7/20\n",
      "3039/3039 [==============================] - 11s 3ms/step - loss: 0.5051 - mae: 0.2774 - mse: 0.1274 - val_loss: 0.8225 - val_mae: 0.2789 - val_mse: 0.1308\n",
      "Epoch 8/20\n",
      "3039/3039 [==============================] - 11s 3ms/step - loss: 0.5049 - mae: 0.2774 - mse: 0.1275 - val_loss: 0.8227 - val_mae: 0.2793 - val_mse: 0.1309\n",
      "Epoch 9/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5043 - mae: 0.2776 - mse: 0.1276 - val_loss: 0.8223 - val_mae: 0.2788 - val_mse: 0.1307\n",
      "Epoch 10/20\n",
      "3039/3039 [==============================] - 10s 3ms/step - loss: 0.5053 - mae: 0.2776 - mse: 0.1278 - val_loss: 0.8223 - val_mae: 0.2784 - val_mse: 0.1302\n",
      "Epoch 11/20\n",
      "3039/3039 [==============================] - 11s 3ms/step - loss: 0.5052 - mae: 0.2775 - mse: 0.1276 - val_loss: 0.8209 - val_mae: 0.2762 - val_mse: 0.1289\n",
      "Epoch 12/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5054 - mae: 0.2777 - mse: 0.1277 - val_loss: 0.8225 - val_mae: 0.2799 - val_mse: 0.1314\n",
      "Epoch 13/20\n",
      "3039/3039 [==============================] - 10s 3ms/step - loss: 0.5050 - mae: 0.2774 - mse: 0.1275 - val_loss: 0.8219 - val_mae: 0.2800 - val_mse: 0.1318\n",
      "Epoch 14/20\n",
      "3039/3039 [==============================] - 11s 3ms/step - loss: 0.5049 - mae: 0.2774 - mse: 0.1275 - val_loss: 0.8222 - val_mae: 0.2788 - val_mse: 0.1309\n",
      "Epoch 15/20\n",
      "3039/3039 [==============================] - 11s 3ms/step - loss: 0.5044 - mae: 0.2773 - mse: 0.1275 - val_loss: 0.8235 - val_mae: 0.2812 - val_mse: 0.1326\n",
      "Epoch 16/20\n",
      "3039/3039 [==============================] - 10s 3ms/step - loss: 0.5052 - mae: 0.2775 - mse: 0.1277 - val_loss: 0.8231 - val_mae: 0.2791 - val_mse: 0.1311\n",
      "Epoch 17/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5047 - mae: 0.2777 - mse: 0.1277 - val_loss: 0.8221 - val_mae: 0.2771 - val_mse: 0.1292\n",
      "Epoch 18/20\n",
      "3039/3039 [==============================] - 11s 3ms/step - loss: 0.5045 - mae: 0.2771 - mse: 0.1273 - val_loss: 0.8241 - val_mae: 0.2826 - val_mse: 0.1337\n",
      "Epoch 19/20\n",
      "3039/3039 [==============================] - 10s 3ms/step - loss: 0.5049 - mae: 0.2775 - mse: 0.1276 - val_loss: 0.8228 - val_mae: 0.2816 - val_mse: 0.1331\n",
      "Epoch 20/20\n",
      "3039/3039 [==============================] - 11s 4ms/step - loss: 0.5055 - mae: 0.2780 - mse: 0.1280 - val_loss: 0.8228 - val_mae: 0.2772 - val_mse: 0.1292\n",
      "0.8209351897239685\n"
     ]
    }
   ],
   "source": [
    "mlpModel.compile(optimizer=Adam(lr=1e-3), loss=CCC_VA, metrics=['mae','mse'])\n",
    "mlpModel.summary()\n",
    "\n",
    "save_best_model = SaveBestModel('val_loss',False)\n",
    "mlpModel.fit(X_train,y_train, batch_size=batch_size, epochs=20, verbose=1, callbacks=[save_best_model], validation_data=(X_val,y_val))\n",
    "best_model_weights = save_best_model.best_model_weights\n",
    "print(save_best_model.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.4298415239084466, 0.4922025796049645, 0.4610220517567055)\n",
      "0.42984280225697313 0.4922040352852844\n",
      "Best weights:\n",
      "(0.42974953297503016, 0.49796161300946734, 0.4638555729922488)\n",
      "0.42975080214878003 0.4979630952849663\n"
     ]
    }
   ],
   "source": [
    "print_ccc()\n",
    "print('Best weights:')\n",
    "mlpModel.set_weights(best_model_weights)\n",
    "print_ccc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.42974953297503016, 0.49796161300946734, 0.4638555729922488)\n",
      "0.42975080214878003 0.4979630952849663\n"
     ]
    }
   ],
   "source": [
    "if False:\n",
    "    mlpModel.save_weights('va_enet0_vgaf.h5') #(0.42974953297503016, 0.49796161300946734, 0.4638555729922488)\n",
    "else:\n",
    "    mlpModel.load_weights('va_enet0_vgaf.h5')\n",
    "    print_ccc()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Smooth validation predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 71/71 [00:10<00:00,  6.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/VA_Estimation_Challenge/Validation_Set')\n",
    "test_videos={}\n",
    "for filename in tqdm(os.listdir(dirpath)):\n",
    "    fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "    if ext.lower()=='.txt':\n",
    "        X,indices,y_true=[],[],[]\n",
    "        with open(os.path.join(dirpath,filename)) as f:\n",
    "            lines = f.read().splitlines()\n",
    "            prev_val=None\n",
    "            for i,line in enumerate(lines):\n",
    "                if i>0:\n",
    "                    imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                    if imagename in filename2featuresAll:\n",
    "                        X.append(filename2featuresAll[imagename][1])\n",
    "                        indices.append(i)\n",
    "                        \n",
    "                        splitted_line=line.split(',')\n",
    "                        valence=float(splitted_line[0])\n",
    "                        arousal=float(splitted_line[1])\n",
    "                        y_true.append((valence,arousal))\n",
    "                        \n",
    "        test_videos[fn]=(mlpModel.predict(np.array(X)),indices,np.array(y_true))\n",
    "print(len(test_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean 0 (0.4297495329925397, 0.4979616130077603, 0.46385557300015)\n",
      "mean 2 (0.43746115316549106, 0.5131634836328756, 0.4753123183991833)\n",
      "median 2 (0.43544421989237614, 0.5072473979359517, 0.47134580891416394)\n",
      "mean 7 (0.44849536890896224, 0.5353431416480636, 0.49191925527851293)\n",
      "median 7 (0.44590019769035444, 0.527161073024317, 0.48653063535733576)\n"
     ]
    }
   ],
   "source": [
    "hyperparams=[(isMean,delta) for delta in [0,2,7]  for isMean in [1,0] if not (isMean==0 and delta==0)]\n",
    "total_true=[]\n",
    "total_preds=[[] for _ in range(len(hyperparams))]\n",
    "for videoname,(y_pred_va,indices,y_true) in test_videos.items():\n",
    "    for i,ind in enumerate(indices):\n",
    "        if y_true[i][0]>=-1 and y_true[i][1]>=-1:\n",
    "            total_true.append(y_true[i])\n",
    "    cur_ind=0\n",
    "    preds=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds.append(y_pred_va[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds.append(y_pred_va[cur_ind])\n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_va[cur_ind-1]+(1-w)*y_pred_va[cur_ind]\n",
    "                preds.append(pred)\n",
    "    \n",
    "    preds=np.array(preds)\n",
    "    for hInd,(isMean,delta) in enumerate(hyperparams):\n",
    "        cur_preds=[]\n",
    "        for i in range(len(preds)):\n",
    "            i1=max(i-delta,0)\n",
    "            if isMean:\n",
    "                pred=np.mean(preds[i1:i+delta+1],axis=0)\n",
    "            else:\n",
    "                pred=np.median(preds[i1:i+delta+1],axis=0)\n",
    "            cur_preds.append(pred)\n",
    "        for i,ind in enumerate(indices):\n",
    "            if y_true[i][0]>=-1 and y_true[i][1]>=-1:\n",
    "                total_preds[hInd].append(cur_preds[ind-1])\n",
    "\n",
    "total_true=np.array(total_true)\n",
    "gt_V=total_true[:,0]\n",
    "gt_A=total_true[:,1]\n",
    "\n",
    "for hInd,(isMean,delta) in enumerate(hyperparams):\n",
    "    preds=np.array(total_preds[hInd])\n",
    "    pred_V=preds[:,0]\n",
    "    pred_A=preds[:,1]\n",
    "    print('mean' if isMean else 'median',delta,metric_for_VA(gt_V,gt_A,pred_V,pred_A))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save test results "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152 ['2-30-640x360', '3-25-1920x1080', '6-30-1920x1080_left', '6-30-1920x1080_right', '11-24-1920x1080']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(DATA_DIR,'Valence_Arousal_Estimation_Challenge_test_set_release.txt'),'r') as f:\n",
    "    test_set_videos=f.read().splitlines()\n",
    "print(len(test_set_videos),test_set_videos[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 564/564 [00:24<00:00, 23.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "152\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "test_videos={}\n",
    "for videoname in tqdm(os.listdir(data_dir)):\n",
    "    if videoname in test_set_videos:\n",
    "        X,indices=[],[]\n",
    "        images=[img_name for img_name in os.listdir(os.path.join(data_dir,videoname)) if img_name.lower().endswith('.jpg')]\n",
    "        for img_name in sorted(images, key=compare_filenames):\n",
    "            k=videoname+'/'+img_name\n",
    "            if k in filename2featuresAll:\n",
    "                X.append(filename2featuresAll[k][1])\n",
    "                indices.append(int(img_name[:-4]))\n",
    "        test_videos[videoname]=(mlpModel.predict(np.array(X)),indices)\n",
    "print(len(test_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "delta=2 #0 2 4 7\n",
    "resdir=os.path.join(DATA_DIR,'test_results/VA/2_mean5') #2_mean5 5_med15\n",
    "if not os.path.exists(resdir):\n",
    "    os.makedirs(resdir)\n",
    "header = 'valence,arousal\\n'\n",
    "\n",
    "for videoname,(y_pred_va,indices) in test_videos.items():\n",
    "    cur_ind=0\n",
    "    preds=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds.append(y_pred_va[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds.append(y_pred_va[cur_ind])\n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_va[cur_ind-1]+(1-w)*y_pred_va[cur_ind]\n",
    "                preds.append(pred)\n",
    "\n",
    "    pred=y_pred_va[cur_ind-1]\n",
    "    vn=videoname\n",
    "    if videoname.endswith('_left'):\n",
    "        vn=videoname[:-5]\n",
    "    elif videoname.endswith('_right'):\n",
    "        vn=videoname[:-6]\n",
    "    for _ in range(indices[-1],video2len[vn]):\n",
    "        preds.append(pred)\n",
    "    preds=np.array(preds)\n",
    "    with open(os.path.join(resdir,videoname+'.txt'), 'w') as f:\n",
    "        f.write(header)\n",
    "        for i in range(len(preds)):\n",
    "            i1=max(i-delta,0)\n",
    "            pred=np.mean(preds[i1:i+delta+1],axis=0)\n",
    "            #pred=np.median(preds[i1:i+delta+1],axis=0)\n",
    "            f.write(str(pred[0])+','+str(pred[1])+'\\n')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AU "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1356861, 1280) (1356861, 12) 2829\n",
      "(445836, 1280) (445836, 12) 9\n"
     ]
    }
   ],
   "source": [
    "def get_image2AU(dirname):\n",
    "    dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/AU_Detection_Challenge/',dirname)\n",
    "    num_missed=0\n",
    "    X,y=[],[]\n",
    "    for filename in os.listdir(dirpath):\n",
    "        fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "        if ext.lower()=='.txt':\n",
    "            with open(os.path.join(dirpath,filename)) as f:\n",
    "                lines = f.read().splitlines()\n",
    "                for i,line in enumerate(lines):\n",
    "                    if i>0:\n",
    "                        splitted_line=line.split(',')\n",
    "                        aus=list(map(int,splitted_line))\n",
    "                        if min(aus)>=0:\n",
    "                            imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                            if imagename in filename2featuresAll:\n",
    "                                X.append(filename2featuresAll[imagename][0])\n",
    "                                y.append(aus)\n",
    "                            else:\n",
    "                                num_missed+=1\n",
    "    X=np.array(X)\n",
    "    y=np.array(y)\n",
    "    print(X.shape,y.shape,num_missed)\n",
    "    return X,y\n",
    "\n",
    "X_train,y_train=get_image2AU('Train_Set')\n",
    "X_val,y_val=get_image2AU('Validation_Set')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_au():\n",
    "    y_val_preds=mlpModel.predict(X_val)\n",
    "    new_pred = ((y_val_preds >= 0.5) * 1)\n",
    "    print(np.mean([f1_score(y_true=y_val[:,i],y_pred=new_pred[:,i]) for i in range(y_val_preds.shape[1])]))\n",
    "    print(f1_score_max(y_val,y_val_preds,thresh=np.arange(0.1,1,0.1)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n",
      "[[ 0.56699737  4.23148962]\n",
      " [ 0.52623698 10.02853659]\n",
      " [ 0.59099307  3.24746184]\n",
      " [ 0.68134051  1.87862194]\n",
      " [ 0.82949071  1.25874673]\n",
      " [ 0.7628283   1.45119134]\n",
      " [ 0.66223812  2.04094492]\n",
      " [ 0.51446878 17.77857704]\n",
      " [ 0.51624061 15.8935131 ]\n",
      " [ 0.51314264 19.52205628]\n",
      " [ 1.3391547   0.79791885]\n",
      " [ 0.54019984  6.71892981]]\n"
     ]
    }
   ],
   "source": [
    "num_labels=y_train.shape[1]\n",
    "print(num_labels)\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "class_weights = np.empty([num_labels, 2])\n",
    "for i in range(num_labels):\n",
    "    neg, pos = np.bincount(y_train[:, i])\n",
    "    total = neg + pos\n",
    "    weight_for_0 = (1 / neg) * (total / 2.0)\n",
    "    weight_for_1 = (1 / pos) * (total / 2.0)\n",
    "\n",
    "    class_weights[i][0]=weight_for_0\n",
    "    class_weights[i][1]=weight_for_1\n",
    "    #class_weights[i] = compute_class_weight('balanced', [0,1], y_train[:, i])\n",
    "print(class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    loss='binary_crossentropy'\n",
    "    #loss='hinge'\n",
    "else:\n",
    "    import tensorflow.keras.backend as K\n",
    "    def get_weighted_loss(weights):\n",
    "        def weighted_loss(y_true, y_pred):\n",
    "            y_true=tf.cast(y_true, tf.float32)\n",
    "            ce=K.binary_crossentropy(y_true, y_pred)\n",
    "            return K.mean((weights[:,0]**(1-y_true))*(weights[:,1]**(y_true))*ce, axis=-1)\n",
    "        return weighted_loss\n",
    "    loss=get_weighted_loss(class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics=[tf.keras.metrics.AUC(multi_label=True,name='auc'), tf.keras.metrics.BinaryAccuracy(),tf.keras.metrics.Recall(),tf.keras.metrics.Precision()] # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=512 #128\n",
    "mlpModel=Sequential()\n",
    "if False:\n",
    "    mlpModel.add(Dense(y_train.shape[1], input_shape=X_train.shape[1:],activation='sigmoid',use_bias=True,kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size)))\n",
    "else:\n",
    "    mlpModel.add(Dense(128, input_shape=X_train.shape[1:],activation='relu')) #256\n",
    "    mlpModel.add(Dense(y_train.shape[1],activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_9 (Dense)              (None, 128)               163968    \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 12)                1548      \n",
      "=================================================================\n",
      "Total params: 165,516\n",
      "Trainable params: 165,516\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/20\n",
      "2651/2651 [==============================] - 18s 6ms/step - loss: 0.3631 - auc: 0.9146 - binary_accuracy: 0.8374 - recall: 0.8133 - precision: 0.5637 - val_loss: 0.6988 - val_auc: 0.8226 - val_binary_accuracy: 0.8227 - val_recall: 0.7469 - val_precision: 0.5523\n",
      "Epoch 2/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.2615 - auc: 0.9556 - binary_accuracy: 0.8854 - recall: 0.8644 - precision: 0.6591 - val_loss: 0.7778 - val_auc: 0.8188 - val_binary_accuracy: 0.8338 - val_recall: 0.7322 - val_precision: 0.5768\n",
      "Epoch 3/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.2381 - auc: 0.9622 - binary_accuracy: 0.8953 - recall: 0.8752 - precision: 0.6827 - val_loss: 0.8079 - val_auc: 0.8175 - val_binary_accuracy: 0.8235 - val_recall: 0.7306 - val_precision: 0.5553\n",
      "Epoch 4/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.2241 - auc: 0.9659 - binary_accuracy: 0.9014 - recall: 0.8808 - precision: 0.6974 - val_loss: 0.8771 - val_auc: 0.8140 - val_binary_accuracy: 0.8280 - val_recall: 0.7483 - val_precision: 0.5629\n",
      "Epoch 5/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.2146 - auc: 0.9682 - binary_accuracy: 0.9055 - recall: 0.8851 - precision: 0.7081 - val_loss: 0.9068 - val_auc: 0.8134 - val_binary_accuracy: 0.8346 - val_recall: 0.7198 - val_precision: 0.5803\n",
      "Epoch 6/20\n",
      "2651/2651 [==============================] - 15s 6ms/step - loss: 0.2068 - auc: 0.9700 - binary_accuracy: 0.9090 - recall: 0.8883 - precision: 0.7171 - val_loss: 0.9475 - val_auc: 0.8064 - val_binary_accuracy: 0.8222 - val_recall: 0.7418 - val_precision: 0.5516\n",
      "Epoch 7/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.2013 - auc: 0.9713 - binary_accuracy: 0.9112 - recall: 0.8908 - precision: 0.7227 - val_loss: 1.0182 - val_auc: 0.8062 - val_binary_accuracy: 0.8325 - val_recall: 0.7382 - val_precision: 0.5733\n",
      "Epoch 8/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1967 - auc: 0.9723 - binary_accuracy: 0.9133 - recall: 0.8924 - precision: 0.7283 - val_loss: 1.0326 - val_auc: 0.8080 - val_binary_accuracy: 0.8399 - val_recall: 0.6817 - val_precision: 0.5987\n",
      "Epoch 9/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1918 - auc: 0.9734 - binary_accuracy: 0.9157 - recall: 0.8946 - precision: 0.7345 - val_loss: 1.0974 - val_auc: 0.8037 - val_binary_accuracy: 0.8369 - val_recall: 0.7254 - val_precision: 0.5846\n",
      "Epoch 10/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1883 - auc: 0.9742 - binary_accuracy: 0.9171 - recall: 0.8961 - precision: 0.7384 - val_loss: 1.1091 - val_auc: 0.8025 - val_binary_accuracy: 0.8367 - val_recall: 0.6968 - val_precision: 0.5883\n",
      "Epoch 11/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1864 - auc: 0.9745 - binary_accuracy: 0.9180 - recall: 0.8972 - precision: 0.7409 - val_loss: 1.1908 - val_auc: 0.8022 - val_binary_accuracy: 0.8448 - val_recall: 0.6929 - val_precision: 0.6091\n",
      "Epoch 12/20\n",
      "2651/2651 [==============================] - 15s 6ms/step - loss: 0.1835 - auc: 0.9752 - binary_accuracy: 0.9193 - recall: 0.8985 - precision: 0.7447 - val_loss: 1.1571 - val_auc: 0.8011 - val_binary_accuracy: 0.8342 - val_recall: 0.7156 - val_precision: 0.5798\n",
      "Epoch 13/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1807 - auc: 0.9758 - binary_accuracy: 0.9206 - recall: 0.8995 - precision: 0.7479 - val_loss: 1.2058 - val_auc: 0.7997 - val_binary_accuracy: 0.8381 - val_recall: 0.6994 - val_precision: 0.5913\n",
      "Epoch 14/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1781 - auc: 0.9763 - binary_accuracy: 0.9217 - recall: 0.9007 - precision: 0.7510 - val_loss: 1.3130 - val_auc: 0.7957 - val_binary_accuracy: 0.8411 - val_recall: 0.7070 - val_precision: 0.5973\n",
      "Epoch 15/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1771 - auc: 0.9765 - binary_accuracy: 0.9222 - recall: 0.9015 - precision: 0.7525 - val_loss: 1.2657 - val_auc: 0.7964 - val_binary_accuracy: 0.8417 - val_recall: 0.6915 - val_precision: 0.6016\n",
      "Epoch 16/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1744 - auc: 0.9770 - binary_accuracy: 0.9234 - recall: 0.9023 - precision: 0.7558 - val_loss: 1.3331 - val_auc: 0.7956 - val_binary_accuracy: 0.8428 - val_recall: 0.6955 - val_precision: 0.6035\n",
      "Epoch 17/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1723 - auc: 0.9774 - binary_accuracy: 0.9243 - recall: 0.9031 - precision: 0.7583 - val_loss: 1.2910 - val_auc: 0.7952 - val_binary_accuracy: 0.8380 - val_recall: 0.6883 - val_precision: 0.5927\n",
      "Epoch 18/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1717 - auc: 0.9776 - binary_accuracy: 0.9245 - recall: 0.9037 - precision: 0.7589 - val_loss: 1.4185 - val_auc: 0.7905 - val_binary_accuracy: 0.8436 - val_recall: 0.6849 - val_precision: 0.6076\n",
      "Epoch 19/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1698 - auc: 0.9779 - binary_accuracy: 0.9255 - recall: 0.9042 - precision: 0.7618 - val_loss: 1.3307 - val_auc: 0.7944 - val_binary_accuracy: 0.8397 - val_recall: 0.6963 - val_precision: 0.5958\n",
      "Epoch 20/20\n",
      "2651/2651 [==============================] - 16s 6ms/step - loss: 0.1683 - auc: 0.9782 - binary_accuracy: 0.9261 - recall: 0.9050 - precision: 0.7633 - val_loss: 1.2886 - val_auc: 0.7930 - val_binary_accuracy: 0.8313 - val_recall: 0.7130 - val_precision: 0.5737\n",
      "0.69876629114151\n"
     ]
    }
   ],
   "source": [
    "mlpModel.compile(optimizer=Adam(lr=1e-3), loss=loss, metrics=metrics)\n",
    "mlpModel.summary()\n",
    "\n",
    "save_best_model = SaveBestModel('val_loss',False)\n",
    "mlpModel.fit(X_train,y_train, batch_size=batch_size, epochs=20, verbose=1, callbacks=[save_best_model], validation_data=(X_val,y_val))\n",
    "best_model_weights = save_best_model.best_model_weights\n",
    "print(save_best_model.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4864278055496279\n",
      "(0.49427122676733243, 0.6416666666666666, array([0.51627374, 0.40038851, 0.49818827, 0.58588789, 0.7150855 ,\n",
      "       0.72659929, 0.69216924, 0.28844102, 0.17527699, 0.18035356,\n",
      "       0.82694271, 0.32564799]), array([0.8, 0.7, 0.7, 0.6, 0.5, 0.6, 0.6, 0.8, 0.9, 0.6, 0.3, 0.6]), array([0.86745799, 0.88922608, 0.84654896, 0.79186741, 0.74430957,\n",
      "       0.79999821, 0.84309701, 0.9461506 , 0.95742605, 0.94134615,\n",
      "       0.7577607 , 0.82674122]))\n",
      "Best weights:\n",
      "0.5018743464268921\n",
      "(0.5326505266567211, 0.6666666666666666, array([0.5586506 , 0.46318824, 0.57471749, 0.61564439, 0.73792177,\n",
      "       0.74323626, 0.71372861, 0.34674426, 0.18797831, 0.2369548 ,\n",
      "       0.83616326, 0.37687833]), array([0.8, 0.8, 0.8, 0.6, 0.5, 0.5, 0.6, 0.8, 0.7, 0.9, 0.3, 0.7]), array([0.88197678, 0.90478562, 0.864181  , 0.79781579, 0.77860469,\n",
      "       0.80720489, 0.85493993, 0.95065226, 0.95263954, 0.95683615,\n",
      "       0.77205968, 0.84885698]))\n"
     ]
    }
   ],
   "source": [
    "print_au()\n",
    "print('Best weights:')\n",
    "mlpModel.set_weights(best_model_weights)\n",
    "print_au()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.507568027779308\n",
      "(0.5367145284583944, 0.65, array([0.56438654, 0.47163507, 0.57207889, 0.61677155, 0.74818918,\n",
      "       0.74534595, 0.71942028, 0.35743029, 0.19360184, 0.24140214,\n",
      "       0.83680711, 0.37350551]), array([0.8, 0.8, 0.7, 0.5, 0.5, 0.5, 0.6, 0.8, 0.8, 0.8, 0.3, 0.7]), array([0.8884859 , 0.90530374, 0.8692748 , 0.79757579, 0.78782557,\n",
      "       0.80682583, 0.8569631 , 0.94949937, 0.94323473, 0.95116814,\n",
      "       0.77594452, 0.85026333]))\n"
     ]
    }
   ],
   "source": [
    "if False:\n",
    "    mlpModel.save_weights('au_enet0_vgaf.h5') #0.507568027779308\n",
    "else:\n",
    "    mlpModel.load_weights('au_enet0_vgaf.h5')\n",
    "    print_au()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5367145284583944\n"
     ]
    }
   ],
   "source": [
    "pred=mlpModel.predict(X_val)\n",
    "#thresholds=0.5\n",
    "#thresholds=np.array([0.7, 0.8, 0.7, 0.6, 0.5, 0.5, 0.6, 0.8, 0.7, 0.8, 0.3, 0.7]) #au_enet0_7 - 0.5228095834398018\n",
    "thresholds=np.array([0.8, 0.8, 0.7, 0.5, 0.5, 0.5, 0.6, 0.8, 0.8, 0.8, 0.3, 0.7]) #vgaf 0.5367145284583944\n",
    "new_pred = ((pred >= thresholds) * 1)\n",
    "print(np.mean([f1_score(y_true=y_val[:,i],y_pred=new_pred[:,i]) for i in range(pred.shape[1])]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mobilenet\n",
    "features\n",
    "logreg: 0.47297223875271305\n",
    "(0.5239126048120428, 0.6166666666666667\n",
    "dense: 0.4776362901110953\n",
    "\n",
    "emotions\n",
    "logreg: 0.43245972823656503\n",
    "0.44210386971177873, 0.5083333333333334\n",
    "dense: 0.4515366088060122\n",
    "(0.4866808135583551, 0.6083333333333333\n",
    "\n",
    "\n",
    "enetb0_7:\n",
    "features\n",
    "logreg: 0.47423963532530294\n",
    "(0.5172051728975023, 0.6\n",
    "dense: 0.4909581965862712\n",
    "(0.5183018786548269, 0.6749999999999999\n",
    "\n",
    "enet_b0_8_best_afew\n",
    "features\n",
    "dense: 0.4850984023504916\n",
    "(0.5154674855367459, 0.6749999999999999\n",
    "\n",
    "enet_b0_8_best_vgaf\n",
    "features\n",
    "dense: 0.507568027779308\n",
    "(0.5367145284583944, 0.65\n",
    "\n",
    "\n",
    "enetb2_8:\n",
    "features\n",
    "logreg: 0.467894360096913\n",
    "(0.5026441356600955, 0.5916666666666666\n",
    "dense: 0.48209903459210973\n",
    "(0.5121049982984248, 0.6666666666666666"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Smooth validation predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 105/105 [00:23<00:00,  4.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "dirpath=os.path.join(DATA_DIR,'Third ABAW Annotations/AU_Detection_Challenge/Validation_Set')\n",
    "test_videos={}\n",
    "for filename in tqdm(os.listdir(dirpath)):\n",
    "    fn, ext = os.path.splitext(os.path.basename(filename))\n",
    "    if ext.lower()=='.txt':\n",
    "        X,indices,y_true=[],[],[]\n",
    "        with open(os.path.join(dirpath,filename)) as f:\n",
    "            lines = f.read().splitlines()\n",
    "            prev_val=None\n",
    "            for i,line in enumerate(lines):\n",
    "                if i>0:\n",
    "                    imagename=fn+'/'+get_names(i)+'.jpg'\n",
    "                    if imagename in filename2featuresAll:\n",
    "                        X.append(filename2featuresAll[imagename][0])\n",
    "                        indices.append(i)\n",
    "                        splitted_line=line.split(',')\n",
    "                        aus=list(map(int,splitted_line))\n",
    "                        y_true.append(aus)\n",
    "        test_videos[fn]=(mlpModel.predict(np.array(X)),indices,np.array(y_true))\n",
    "print(len(test_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mean 0 0.5367145284583944\n",
      "mean 2 0.5446927222162641\n",
      "median 2 0.5429543082968176\n",
      "mean 7 0.5445119128957571\n",
      "median 7 0.5478115868229798\n"
     ]
    }
   ],
   "source": [
    "hyperparams=[(isMean,delta) for delta in [0,2,7]  for isMean in [1,0] if not (isMean==0 and delta==0)]\n",
    "total_true=[]\n",
    "total_preds=[[] for _ in range(len(hyperparams))]\n",
    "for videoname,(y_pred_aus,indices,y_true) in test_videos.items():\n",
    "    for i,ind in enumerate(indices):\n",
    "        if min(y_true[i])>=0:\n",
    "            total_true.append(y_true[i])\n",
    "    cur_ind=0\n",
    "    preds=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds.append(y_pred_aus[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds.append(y_pred_aus[cur_ind])\n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_aus[cur_ind-1]+(1-w)*y_pred_aus[cur_ind]\n",
    "                preds.append(pred)\n",
    "    \n",
    "    preds=np.array(preds)\n",
    "    for hInd,(isMean,delta) in enumerate(hyperparams):\n",
    "        cur_preds=[]\n",
    "        for i in range(len(preds)):\n",
    "            i1=max(i-delta,0)\n",
    "            if isMean:\n",
    "                pred=np.mean(preds[i1:i+delta+1],axis=0)\n",
    "            else:\n",
    "                pred=np.median(preds[i1:i+delta+1],axis=0)\n",
    "            aus=(pred>=thresholds)*1\n",
    "            cur_preds.append(aus)\n",
    "        for i,ind in enumerate(indices):\n",
    "            if min(y_true[i])>=0:\n",
    "                total_preds[hInd].append(cur_preds[ind-1])\n",
    "\n",
    "total_true=np.array(total_true)\n",
    "for hInd,(isMean,delta) in enumerate(hyperparams):\n",
    "    preds=np.array(total_preds[hInd])\n",
    "    print('mean' if isMean else 'median',delta,np.mean([f1_score(y_true=total_true[:,i],y_pred=preds[:,i]) for i in range(preds.shape[1])]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save test results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141 ['2-30-640x360', '3-25-1920x1080', '6-30-1920x1080_left', '6-30-1920x1080_right', '11-24-1920x1080']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(DATA_DIR,'Action_Unit_Detection_Challenge_test_set_release.txt'),'r') as f:\n",
    "    test_set_videos=f.read().splitlines()\n",
    "print(len(test_set_videos),test_set_videos[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 564/564 [00:36<00:00, 15.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "141\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "data_dir=os.path.join(DATA_DIR,'faces')\n",
    "test_videos={}\n",
    "for videoname in tqdm(os.listdir(data_dir)):\n",
    "    if videoname in test_set_videos:\n",
    "        X,indices=[],[]\n",
    "        images=[img_name for img_name in os.listdir(os.path.join(data_dir,videoname)) if img_name.lower().endswith('.jpg')]\n",
    "        for img_name in sorted(images, key=compare_filenames):\n",
    "            k=videoname+'/'+img_name\n",
    "            if k in filename2featuresAll:\n",
    "                X.append(filename2featuresAll[k][0])\n",
    "                indices.append(int(img_name[:-4]))\n",
    "        test_videos[videoname]=(mlpModel.predict(np.array(X)),indices)\n",
    "print(len(test_videos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 141/141 [00:10<00:00, 13.28it/s]\n"
     ]
    }
   ],
   "source": [
    "delta=0 #0 2 4 7\n",
    "resdir=os.path.join(DATA_DIR,'test_results/AU/1_single') #2_mean5 5_med15\n",
    "if not os.path.exists(resdir):\n",
    "    os.makedirs(resdir)\n",
    "header = 'AU1,AU2,AU4,AU6,AU7,AU10,AU12,AU15,AU23,AU24,AU25,AU26\\n'\n",
    "\n",
    "for videoname,(y_pred_aus,indices) in tqdm(test_videos.items()):\n",
    "    cur_ind=0\n",
    "    preds=[]\n",
    "    for i in range(indices[-1]):\n",
    "        if indices[cur_ind]-1==i:\n",
    "            preds.append(y_pred_aus[cur_ind])\n",
    "            cur_ind+=1\n",
    "        else:\n",
    "            if cur_ind==0:\n",
    "                preds.append(y_pred_aus[cur_ind])\n",
    "            \n",
    "            else:\n",
    "                w=(i-indices[cur_ind-1]+1)/(indices[cur_ind]-indices[cur_ind-1])\n",
    "                pred=w*y_pred_aus[cur_ind-1]+(1-w)*y_pred_aus[cur_ind]\n",
    "                preds.append(pred)\n",
    "                \n",
    "    pred=y_pred_aus[cur_ind-1]\n",
    "    vn=videoname\n",
    "    if videoname.endswith('_left'):\n",
    "        vn=videoname[:-5]\n",
    "    elif videoname.endswith('_right'):\n",
    "        vn=videoname[:-6]\n",
    "    for _ in range(indices[-1],video2len[vn]):\n",
    "        preds.append(pred)\n",
    "    \n",
    "    preds=np.array(preds)\n",
    "    with open(os.path.join(resdir,videoname+'.txt'), 'w') as f:\n",
    "        f.write(header)\n",
    "        for i in range(len(preds)):\n",
    "            i1=max(i-delta,0)\n",
    "            pred=np.mean(preds[i1:i+delta+1],axis=0)\n",
    "            #pred=np.median(preds[i1:i+delta+1],axis=0)\n",
    "            aus=(pred>=thresholds)*1\n",
    "            f.write(','.join(map(str,aus))+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-task challenge "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(142225, 1416) (142225, 2) (142225,) (142225, 12) (142225,) 2666\n",
      "(26876, 1416) (26876, 2) (26876,) (26876, 12) (26876,) 211\n"
     ]
    }
   ],
   "source": [
    "def get_image2all(filename):\n",
    "    with open(os.path.join(DATA_DIR,'Third ABAW Annotations/MTL_Challenge/'+filename)) as f:\n",
    "        mtl_lines = f.read().splitlines()\n",
    "    num_missed=0\n",
    "    X,y_va,y_expr,y_aus=[],[],[],[]\n",
    "    masks_va,masks_expr,masks_aus=[],[],[]\n",
    "    for line in mtl_lines[1:]:\n",
    "        splitted_line=line.split(',')\n",
    "        imagename=splitted_line[0]\n",
    "        valence=float(splitted_line[1])\n",
    "        arousal=float(splitted_line[2])\n",
    "        expression=int(splitted_line[3])\n",
    "        aus=list(map(int,splitted_line[4:]))\n",
    "        \n",
    "        mask_VA=(valence>-5 and arousal>-5)\n",
    "        if not mask_VA:\n",
    "            valence=arousal=0\n",
    "            \n",
    "        mask_expr=(expression>-1)\n",
    "        if not mask_expr:\n",
    "            expression=0\n",
    "            \n",
    "        mask_aus=min(aus)>=0\n",
    "        if not mask_aus:\n",
    "            aus=[0]*len(aus)\n",
    "        if mask_VA or mask_expr or mask_aus:\n",
    "            if imagename in filename2featuresAll:\n",
    "                #X.append(filename2featuresAll[imagename][0])\n",
    "                X.append(np.concatenate((filename2featuresAll[imagename][0],filename2featuresAll[imagename][1])))\n",
    "                y_va.append((valence,arousal))\n",
    "                masks_va.append(mask_VA)\n",
    "                \n",
    "                y_expr.append(expression)\n",
    "                masks_expr.append(mask_expr)\n",
    "                \n",
    "                y_aus.append(aus)\n",
    "                masks_aus.append(mask_aus)\n",
    "            else:\n",
    "                num_missed+=1\n",
    "    X=np.array(X)\n",
    "    y_va=np.array(y_va)\n",
    "    y_expr=np.array(y_expr)\n",
    "    y_aus=np.array(y_aus)\n",
    "    masks_va=np.array(masks_va).astype(np.float32)\n",
    "    masks_expr=np.array(masks_expr).astype(np.float32)\n",
    "    masks_aus=np.array(masks_aus).astype(np.float32)\n",
    "    print(X.shape,y_va.shape,y_expr.shape,y_aus.shape,masks_va.shape,num_missed)\n",
    "    return X,y_va,y_expr,y_aus,masks_va,masks_expr,masks_aus\n",
    "\n",
    "X_train,y_va_train,y_expr_train,y_aus_train,masks_va_train,masks_expr_train,masks_aus_train=get_image2all('train_set.txt')\n",
    "X_val,y_va_val,y_expr_val,y_aus_val,masks_va_val,masks_expr_val,masks_aus_val=get_image2all('validation_set.txt')\n",
    "TRAIN_VAL=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(169101, 1416) (169101, 2) (169101,) (169101, 12) (169101,) (169101,) (169101,)\n"
     ]
    }
   ],
   "source": [
    "if False:\n",
    "    TRAIN_VAL=True\n",
    "    X_train=np.concatenate((X_train,X_val))\n",
    "    y_va_train=np.concatenate((y_va_train,y_va_val))\n",
    "    y_expr_train=np.concatenate((y_expr_train,y_expr_val))\n",
    "    y_aus_train=np.concatenate((y_aus_train,y_aus_val))\n",
    "    masks_va_train=np.concatenate((masks_va_train,masks_va_val))\n",
    "    masks_expr_train=np.concatenate((masks_expr_train,masks_expr_val))\n",
    "    masks_aus_train=np.concatenate((masks_aus_train,masks_aus_val))\n",
    "    print(X_train.shape,y_va_train.shape,y_expr_train.shape,y_aus_train.shape,\n",
    "          masks_va_train.shape,masks_expr_train.shape,masks_aus_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25844  5042  3857  4376 21892  9375  6261 29453] {0: 1.1396455657019038, 1: 5.841531138437128, 2: 7.636245786880996, 3: 6.730575868372943, 4: 1.345377306778732, 5: 3.1416533333333336, 6: 4.7042006069318, 7: 1.0} 8 [0 1 2 3 4 5 6 7]\n",
      "[1.13964557 5.84153114 7.63624579 6.73057587 1.34537731 3.14165333\n",
      " 4.70420061 1.        ]\n"
     ]
    }
   ],
   "source": [
    "(unique, counts) = np.unique(y_expr_train[masks_expr_train==1.].astype(int), return_counts=True)\n",
    "num_classes=len(unique)\n",
    "emo_cw=1/counts\n",
    "emo_cw/=emo_cw.min()\n",
    "emo_class_weights = {i:cwi for i,cwi in zip(unique,emo_cw)}\n",
    "print(counts, emo_class_weights, num_classes, unique)\n",
    "print(emo_cw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n",
      "[[ 0.61478236  2.67803499]\n",
      " [ 0.54767511  5.74382689]\n",
      " [ 0.62284785  2.53503769]\n",
      " [ 0.71610871  1.6568252 ]\n",
      " [ 0.8608845   1.19274241]\n",
      " [ 0.80939227  1.30803571]\n",
      " [ 0.68422579  1.85703045]\n",
      " [ 0.51277199 20.0740798 ]\n",
      " [ 0.51470367 17.50256203]\n",
      " [ 0.52343372 11.16838754]\n",
      " [ 1.65902758  0.71569806]\n",
      " [ 0.55872706  4.7569816 ]]\n"
     ]
    }
   ],
   "source": [
    "num_labels=y_aus_train.shape[1]\n",
    "print(num_labels)\n",
    "from sklearn.utils.class_weight import compute_class_weight\n",
    "aus_class_weights = np.empty([num_labels, 2])\n",
    "for i in range(num_labels):\n",
    "    neg, pos = np.bincount(y_aus_train[masks_aus_train==1., i])\n",
    "    total = neg + pos\n",
    "    weight_for_0 = (1 / neg) * (total / 2.0)\n",
    "    weight_for_1 = (1 / pos) * (total / 2.0)\n",
    "\n",
    "    aus_class_weights[i][0]=weight_for_0\n",
    "    aus_class_weights[i][1]=weight_for_1\n",
    "print(aus_class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_va(y_true, y_pred):\n",
    "    res=1-0.5*(CCC(y_true[:,0], y_pred[:,0])+CCC(y_true[:,1], y_pred[:,1]))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WeightedExprSCCE(tf.keras.losses.Loss):\n",
    "    def __init__(self, class_weight, from_logits=False, name='expr_scce'):\n",
    "        if class_weight is None or all(v == 1. for v in class_weight):\n",
    "            self.class_weight = None\n",
    "        else:\n",
    "            self.class_weight = tf.convert_to_tensor(class_weight,\n",
    "                dtype=tf.float32)\n",
    "        self.reduction = tf.keras.losses.Reduction.NONE\n",
    "        self.unreduced_scce = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "            from_logits=from_logits, name=name,\n",
    "            reduction=self.reduction)\n",
    "\n",
    "    def __call__(self, y_true, y_pred, sample_weight=None):\n",
    "        loss = self.unreduced_scce(y_true, y_pred, sample_weight)\n",
    "        if self.class_weight is not None:\n",
    "            weight_mask = tf.gather(self.class_weight, y_true)\n",
    "            loss = tf.math.multiply(loss, weight_mask)\n",
    "        loss=K.mean(loss)\n",
    "        return loss\n",
    "loss_expr=WeightedExprSCCE(emo_cw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_weighted_loss_aus(weights):\n",
    "    def weighted_loss(y_true, y_pred):\n",
    "        y_true=tf.cast(y_true, tf.float32)\n",
    "        ce=K.binary_crossentropy(y_true, y_pred)\n",
    "        res=K.mean((weights[:,0]**(1-y_true))*(weights[:,1]**(y_true))*ce)\n",
    "        return res\n",
    "    return weighted_loss\n",
    "loss_aus=get_weighted_loss_aus(aus_class_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics=[tf.keras.metrics.AUC(multi_label=True,name='auc'), tf.keras.metrics.BinaryAccuracy(),tf.keras.metrics.Recall(),tf.keras.metrics.Precision()] # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=256 #128\n",
    "img = tf.keras.Input(shape=X_train.shape[1:])\n",
    "mask1 = tf.keras.Input(shape=(1,))\n",
    "mask2 = tf.keras.Input(shape=(1,))\n",
    "mask3 = tf.keras.Input(shape=(1,))\n",
    "x=img\n",
    "#x=Dense(128, activation='relu',kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size))(x)\n",
    "va_out = Dense(2, activation='tanh',kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size))(x)\n",
    "va_out_masked=tf.keras.layers.Multiply(name='va_out')([va_out,mask1])\n",
    "\n",
    "#x=va_out\n",
    "\n",
    "expr_out=Dense(8, activation='softmax',kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size))(x)\n",
    "expr_out_masked=tf.keras.layers.Multiply(name='expr_out')([expr_out,mask2])\n",
    "aus_out=Dense(12, activation='sigmoid',kernel_regularizer=tf.keras.regularizers.l2(1.0/batch_size))(x)\n",
    "aus_out_masked=tf.keras.layers.Multiply(name='aus_out')([aus_out,mask3])\n",
    "\n",
    "mtlModel=tf.keras.Model(inputs=[img,mask1,mask2,mask3], outputs=[va_out_masked, expr_out_masked,aus_out_masked])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            [(None, 1416)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 2)            2834        input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_6 (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 8)            11336       input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_7 (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 12)           17004       input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_8 (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "va_out (Multiply)               (None, 2)            0           dense_3[0][0]                    \n",
      "                                                                 input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "expr_out (Multiply)             (None, 8)            0           dense_4[0][0]                    \n",
      "                                                                 input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "aus_out (Multiply)              (None, 12)           0           dense_5[0][0]                    \n",
      "                                                                 input_8[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 31,174\n",
      "Trainable params: 31,174\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "661/661 [==============================] - 9s 12ms/step - loss: 3.9772 - va_out_loss: 0.4375 - expr_out_loss: 2.9862 - aus_out_loss: 0.4333 - va_out_mean_absolute_error: 0.2163 - expr_out_accuracy: 0.6836 - aus_out_auc: 0.8490 - aus_out_binary_accuracy: 0.7983 - aus_out_recall_1: 0.6974 - aus_out_precision_1: 0.4532 - val_loss: 4.7411 - val_va_out_loss: 0.7517 - val_expr_out_loss: 3.2771 - val_aus_out_loss: 0.5671 - val_va_out_mean_absolute_error: 0.2612 - val_expr_out_accuracy: 0.7045 - val_aus_out_auc: 0.7763 - val_aus_out_binary_accuracy: 0.7332 - val_aus_out_recall_1: 0.6816 - val_aus_out_precision_1: 0.4783\n",
      "4.7411065101623535\n"
     ]
    }
   ],
   "source": [
    "mtlModel.compile(optimizer=Adam(lr=1e-3), loss=[loss_va,loss_expr,loss_aus], metrics=[[\"mean_absolute_error\"], [\"accuracy\"],metrics])\n",
    "mtlModel.summary()\n",
    "\n",
    "save_best_model = SaveBestModel('val_loss',False)\n",
    "mtlModel.fit([X_train,masks_va_train,masks_expr_train,masks_aus_train],\n",
    "             [y_va_train,y_expr_train,y_aus_train], batch_size=batch_size, epochs=(1 if TRAIN_VAL else 20), \n",
    "             verbose=1, callbacks=[save_best_model], \n",
    "             validation_data=([X_val,masks_va_val,masks_expr_val,masks_aus_val],[y_va_val,y_expr_val,y_aus_val]))\n",
    "best_model_weights = save_best_model.best_model_weights\n",
    "print(save_best_model.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_all():\n",
    "    y_pred_va,y_pred_expr,y_pred_aus=mtlModel.predict([X_val,masks_va_val,masks_expr_val,masks_aus_val])\n",
    "    print('\\nAV')\n",
    "    gt_V=y_va_val[masks_va_val==1,0]\n",
    "    gt_A=y_va_val[masks_va_val==1,1]\n",
    "    pred_V=y_pred_va[masks_va_val==1,0]\n",
    "    pred_A=y_pred_va[masks_va_val==1,1]\n",
    "    ccc_V,ccc_A,ccc_VA=metric_for_VA(gt_V,gt_A,pred_V,pred_A)\n",
    "    print(gt_V.shape,ccc_V,ccc_A,ccc_VA)\n",
    "    \n",
    "    print('\\nExpression')\n",
    "    print(y_expr_val[masks_expr_val==1].shape)\n",
    "    y_pred=np.argmax(y_pred_expr,axis=1)\n",
    "    print((y_pred==y_expr_val)[masks_expr_val==1].mean())\n",
    "    f1_expr=f1_score(y_true=y_expr_val[masks_expr_val==1],y_pred=y_pred[masks_expr_val==1], average=\"macro\")\n",
    "    print(f1_expr)\n",
    "    print(metric_for_Exp(y_expr_val[masks_expr_val==1],y_pred[masks_expr_val==1]))\n",
    "    \n",
    "    print('\\nAUs')\n",
    "    new_pred = ((y_pred_aus >= 0.5) * 1)\n",
    "    print(new_pred[masks_aus_val==1,:].shape)\n",
    "    f1_au=np.mean([f1_score(y_true=y_aus_val[masks_aus_val==1,i],y_pred=new_pred[masks_aus_val==1,i]) for i in range(y_pred_aus.shape[1])])\n",
    "    print(f1_au)\n",
    "    print(f1_score_max(y_aus_val[masks_aus_val==1,:],y_pred_aus[masks_aus_val==1,:],thresh=np.arange(0.1,1,0.1)))\n",
    "    \n",
    "    total=ccc_VA+f1_expr+f1_au\n",
    "    print('\\nTotal',ccc_VA,f1_expr,f1_au,total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "AV\n",
      "(26876,) 0.5544011376994288 0.4413964265185361 0.4978987821089824\n",
      "\n",
      "Expression\n",
      "(15440,)\n",
      "0.48555699481865283\n",
      "0.4408159586324132\n",
      "(0.4408159586324132, 0.48555699481865283, [0.40022913881993505, 0.3656227239621267, 0.5551684088269454, 0.2686046511627907, 0.5529940961484398, 0.5714285714285714, 0.26362221069019204, 0.5488578680203046])\n",
      "\n",
      "AUs\n",
      "(26876, 12)\n",
      "0.46332223380609444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.49661811530312255, 0.5416666666666666, array([0.53615852, 0.42304198, 0.57974368, 0.56890792, 0.73536316,\n",
      "       0.71222251, 0.65327814, 0.11781089, 0.11587744, 0.27760252,\n",
      "       0.87318841, 0.36622222]), array([0.5, 0.7, 0.6, 0.4, 0.4, 0.4, 0.5, 0.6, 0.6, 0.8, 0.3, 0.7]), array([0.76134841, 0.88186486, 0.82430421, 0.6254651 , 0.71055961,\n",
      "       0.71011311, 0.76938533, 0.76428784, 0.94095103, 0.94887632,\n",
      "       0.79163566, 0.86735377]))\n",
      "\n",
      "Total 0.4978987821089824 0.4408159586324132 0.46332223380609444 1.40203697454749\n"
     ]
    }
   ],
   "source": [
    "print_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best weights:\n",
      "\n",
      "AV\n",
      "(26876,) 0.4218102556513553 0.3570151061777144 0.38941268091453485\n",
      "\n",
      "Expression\n",
      "(15440,)\n",
      "0.3701424870466321\n",
      "0.2966496393771772\n",
      "(0.2966496393771772, 0.3701424870466321, [0.34612162841346955, 0.335718932986337, 0.24719101123595505, 0.06359945872801083, 0.4780876494023904, 0.33641404805914976, 0.14922048997772827, 0.4168438962143769])\n",
      "\n",
      "AUs\n",
      "(26876, 12)\n",
      "0.46193077632133167\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.8/dist-packages/sklearn/metrics/_classification.py:1248: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.48747842211810827, 0.5333333333333332, array([0.5322594 , 0.42471679, 0.56885418, 0.55571648, 0.73383967,\n",
      "       0.70895689, 0.64713913, 0.10933941, 0.07315358, 0.26336249,\n",
      "       0.8707551 , 0.36164794]), array([0.5, 0.7, 0.5, 0.4, 0.4, 0.4, 0.6, 0.6, 0.6, 0.8, 0.3, 0.6]), array([0.7904078 , 0.89607829, 0.78635214, 0.62334425, 0.70140646,\n",
      "       0.7091085 , 0.78843578, 0.79632386, 0.8529171 , 0.9435928 ,\n",
      "       0.78843578, 0.84145706]))\n",
      "\n",
      "Total 0.38941268091453485 0.2966496393771772 0.46193077632133167 1.1479930966130436\n"
     ]
    }
   ],
   "source": [
    "print('Best weights:')\n",
    "mtlModel.set_weights(best_model_weights)\n",
    "print_all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    #mtlModel.save_weights('mtl_enet2_8.h5') #0.3837151018856935 0.3022424974391059 0.46080879636948063 1.1467663956942802\n",
    "    mtlModel.save_weights('mtl_enet2_8.h5') #0.38941268091453485 0.2966496393771772 0.46193077632133167 1.1479930966130436\n",
    "    #mtlModel.save_weights('mtl_enet0_vgaf.h5') #0.38155559224902114 0.2950259083661194 0.4574066636993808 1.1339881643145213\n",
    "else:\n",
    "    mtlModel.load_weights('mtl_enet2_8_orig.h5')\n",
    "    print_all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "mobilenet\n",
    "output layers only: 0.3575652288180158 0.274931498498287 0.4667221169978419 1.0992188443141446\n",
    "0.35813183800284104 0.2823732083688107 0.47113929424277273 1.1116443406144245\n",
    "dense layer: 0.3212823862697798 0.25187087483194115 0.46361689956738755 1.0367701606691084\n",
    "\n",
    "enet0_7\n",
    "output layers only:0.37147839426933726 0.2862848390733215 0.4561286864943848 1.1138919198370436\n",
    "dense layer: 0.37387452943730165 0.2522474413566924 0.4641164043127432 1.0902383751067373\n",
    "not best 0.3944477107433102 0.23835206524054334 0.4603934421520726 1.093193218135926\n",
    "\n",
    "enet_b0_8_best_afew\n",
    "features+emotions\n",
    "output layers only:0.4026537613423008 0.248547670046962 0.45890535417538064 1.1101067855646434\n",
    "\n",
    "\n",
    "enet_b0_8_best_vgaf\n",
    "features+emotions\n",
    "output layers only:0.38615587195723955 0.282608775233668 0.4545389225496568 1.1233035697405644\n",
    "dense: 0.3808455198959431 0.27195008480301336 0.46871028488210437 1.1215058895810608\n",
    "\n",
    "\n",
    "enet2_8\n",
    "output layers only: 0.38307364079552575 0.28723802667892157 0.4593295818373552 1.1296412493118027\n",
    "dense layer: 0.3966562712594687 0.281604125663917 0.45615746671167773 1.1344178636350635\n",
    "\n",
    "features+emotions\n",
    "0.3837151018856935 0.3022424974391059 0.46080879636948063 1.1467663956942802\n",
    "+dense: 0.3779975030622512 0.2987745729052591 0.45822890901489305 1.1350009849824034\n",
    "\n",
    "\n",
    "enet2_7\n",
    "features\n",
    "output layers only: 0.39352969999919685 0.2741800561672634 0.4611839520090892 1.1288937081755495\n",
    "dense layer: 0.3678038228570031 0.2800059725435074 0.4564833238081867 1.104293119208697\n",
    "\n",
    "features+emotions\n",
    "output layers only: 0.38307364079552575 0.28723802667892157 0.4593295818373552 1.1296412493118027\n",
    "dense layer:  0.3919721221529454 0.2821609899822945 0.460412089791669 1.1345452019269089"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.49562541673982846\n"
     ]
    }
   ],
   "source": [
    "if True:\n",
    "    y_pred_va,y_pred_expr,y_pred_aus=mtlModel.predict([X_val,masks_va_val,masks_expr_val,masks_aus_val])\n",
    "#thresholds=np.array([0.5, 0.7, 0.5, 0.4, 0.4, 0.5, 0.5, 0.6, 0.6, 0.8, 0.3, 0.7]) #enet2\n",
    "thresholds=np.array([0.5, 0.7, 0.5, 0.4, 0.4, 0.4, 0.6, 0.6, 0.6, 0.8, 0.3, 0.6]) #enet2\n",
    "#thresholds=np.array([0.6, 0.7, 0.6, 0.4, 0.4, 0.5, 0.5, 0.6, 0.6, 0.7, 0.2, 0.7]) #enet0\n",
    "new_pred = ((y_pred_aus >= thresholds) * 1)\n",
    "f1_au=np.mean([f1_score(y_true=y_aus_val[masks_aus_val==1,i],y_pred=new_pred[masks_aus_val==1,i]) for i in range(y_pred_aus.shape[1])])\n",
    "print(f1_au)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save test results "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "    train_val_videos=set()\n",
    "    for filename in ['train_set.txt', 'validation_set.txt']:\n",
    "        with open(os.path.join(DATA_DIR,'Third ABAW Annotations/MTL_Challenge/'+filename)) as f:\n",
    "            mtl_lines = f.read().splitlines()\n",
    "        for line in mtl_lines[1:]:\n",
    "            splitted_line=line.split(',')\n",
    "            imagename=splitted_line[0]\n",
    "            videoname=imagename.split('/')[0]\n",
    "            train_val_videos.add(videoname)\n",
    "        print(len(train_val_videos))\n",
    "    \n",
    "    data_dir=os.path.join(DATA_DIR,'faces')\n",
    "    test_videos={}\n",
    "    for videoname in os.listdir(data_dir):\n",
    "        if videoname not in train_val_videos:\n",
    "            X,indices,filenames=[],[],[]\n",
    "            images=[img_name for img_name in os.listdir(os.path.join(data_dir,videoname)) if img_name.lower().endswith('.jpg')]\n",
    "            for img_name in sorted(images, key=compare_filenames):\n",
    "                k=videoname+'/'+img_name\n",
    "                if k in filename2featuresAll:\n",
    "                    X.append(np.concatenate((filename2featuresAll[k][0],filename2featuresAll[k][1])))\n",
    "                    indices.append(int(img_name[:-4]))\n",
    "                    filenames.append(k)\n",
    "            test_videos[videoname]=(np.array(X),indices,filenames)\n",
    "    print(len(test_videos))\n",
    "    \n",
    "    resdir=os.path.join(DATA_DIR,'test_results/MTL')\n",
    "    header = 'image,valence,arousal,expression,aus\\n'\n",
    "    with open(os.path.join(resdir,'MTL_predictions.txt'), 'w') as f:\n",
    "        f.write(header)\n",
    "        for videoname,(X,indices,filenames) in test_videos.items():\n",
    "            mask_ones=np.ones(len(X))\n",
    "            y_pred_va,y_pred_expr,y_pred_aus=mtlModel.predict([X,mask_ones,mask_ones,mask_ones])\n",
    "            y_expr=np.argmin(y_pred_expr,axis=1)\n",
    "            y_aus=(y_pred_aus>=thresholds)*1\n",
    "\n",
    "            for filename, va,expr,aus in zip(filenames,y_pred_va,y_expr,y_aus):\n",
    "                s=','.join([filename,str(va[0]),str(va[1]),str(expr),*map(str,aus)])+'\\n'\n",
    "                f.write(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51246 ['image,valence,arousal,expression,aus', 'video45_1/00001.jpg', 'video45_1/00008.jpg', 'video45_1/00017.jpg', 'video45_1/00021.jpg']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(DATA_DIR,'Multi_Task_Learning_Challenge_test_set_release.txt'),'r') as f:\n",
    "    test_set_videos=f.read().splitlines()\n",
    "print(len(test_set_videos),test_set_videos[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "resdir=os.path.join(DATA_DIR,'test_results/MTL/4_e2')\n",
    "if not os.path.exists(resdir):\n",
    "    os.makedirs(resdir)\n",
    "header = 'image,valence,arousal,expression,aus\\n'\n",
    "with open(os.path.join(resdir,'MTL_predictions.txt'), 'w') as f:\n",
    "    f.write(header)\n",
    "    X,filenames=[],[]\n",
    "    mask_ones=np.ones(batch_size)\n",
    "    \n",
    "    for k in test_set_videos[1:]:\n",
    "        if k in filename2featuresAll:\n",
    "            X.append(np.concatenate((filename2featuresAll[k][0],filename2featuresAll[k][1])))\n",
    "            filenames.append(k)\n",
    "            \n",
    "            if len(X)==batch_size:\n",
    "                y_pred_va,y_pred_expr,y_pred_aus=mtlModel.predict([np.array(X),mask_ones,mask_ones,mask_ones])\n",
    "                y_expr=np.argmin(y_pred_expr,axis=1)\n",
    "                y_aus=(y_pred_aus>=thresholds)*1\n",
    "\n",
    "                for filename, va,expr,aus in zip(filenames,y_pred_va,y_expr,y_aus):\n",
    "                    s=','.join([filename,str(va[0]),str(va[1]),str(expr),*map(str,aus)])+'\\n'\n",
    "                    f.write(s)\n",
    "                X,filenames=[],[]\n",
    "        else:\n",
    "            print(k, 'not found')\n",
    "    if len(X)>0:\n",
    "        mask_ones=np.ones(len(X))\n",
    "        y_pred_va,y_pred_expr,y_pred_aus=mtlModel.predict([np.array(X),mask_ones,mask_ones,mask_ones])\n",
    "        y_expr=np.argmin(y_pred_expr,axis=1)\n",
    "        y_aus=(y_pred_aus>=thresholds)*1\n",
    "\n",
    "        for filename, va,expr,aus in zip(filenames,y_pred_va,y_expr,y_aus):\n",
    "            s=','.join([filename,str(va[0]),str(va[1]),str(expr),*map(str,aus)])+'\\n'\n",
    "            f.write(s)\n",
    "        X,filenames=[],[]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}