code/03_ca_extract_facial_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3 Extract facial data from videos.\n",
    "\n",
    "We use a range of libraries to extract facial data from the videos. The main library is [DeepFace](https://github.com/serengil/deepface) but we also considered FER - [Facial Expression Recognition](https://github.com/justinshenk/fer).\n",
    "\n",
    "`pip install deepface`\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1 DeepFace\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from deepface import DeepFace\n",
    "import pandas as pd\n",
    "import pprint\n",
    "import cv2\n",
    "import os\n",
    "import time\n",
    "import utils\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "videos_in = r\"..\\LookitLaughter.test\"\n",
    "demo_data = r\"..\\data\\demo\"\n",
    "data_out = r\"..\\data\\1_interim\"\n",
    "\n",
    "\n",
    "test_video = \"2UWdXP.joke1.rep2.take1.Peekaboo.mp4\"\n",
    "test_video = \"6c6MZQ.joke1.rep1.take1.ThatsNotAHat.mp4\"\n",
    "\n",
    "test_image = \"peekaboo.png\"\n",
    "test_image2 = \"mother-and-baby.jpg\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#just playing around to try out different models built into deepface\n",
    "backends = [\n",
    "  'opencv', \n",
    "  'ssd', \n",
    "  'dlib', \n",
    "  'mtcnn', \n",
    "  'retinaface', \n",
    "  'mediapipe',\n",
    "  'yolov8',\n",
    "  'yunet',\n",
    "]\n",
    "imagepath = os.path.join(demo_data,test_image2)\n",
    "result = DeepFace.analyze(img_path=imagepath, enforce_detection = False, detector_backend = backends[1])\n",
    "\n",
    "for r in result:\n",
    "    pprint.pprint(r)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#show image\n",
    "image1 = cv2.imread(imagepath)\n",
    "plt.imshow(image1)\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getfacedataforvideo(video_path, backend = 'ssd'):\n",
    "    print(\"Processing video: \", video_path)\n",
    "    cap = cv2.VideoCapture(video_path)\n",
    "    success, image = cap.read()\n",
    "    if not success:\n",
    "        print(\"Failed to read video: \", video_path)\n",
    "        return None\n",
    "    facesdf = utils.createfacesdf()\n",
    "    frameidx = 0\n",
    "    fails = []\n",
    "    while success:\n",
    "        try:  \n",
    "            faces = DeepFace.analyze(img_path = image, \n",
    "                                    enforce_detection = True, \n",
    "                                    actions = ('age','gender','emotion'),\n",
    "                                    detector_backend = backend)\n",
    "        except:\n",
    "            #print(f\"deepface.analyse failed for frame {frame}\" )\n",
    "            fails.append(frameidx)\n",
    "            faces = []\n",
    "        if len(faces) > 0:\n",
    "            facesdf = utils.addfacestodf(facesdf,frameidx,faces)\n",
    "        success,image = cap.read()\n",
    "        frameidx += 1\n",
    "    \n",
    "    print(f\"Failed to process {len(fails)}/{frameidx} ({(round(100*len(fails)/frameidx, 1))}%) frames\")\n",
    "    cap.release()\n",
    "    return facesdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "processedvideos = utils.getprocessedvideos(data_out)\n",
    "processedvideos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "forceFaces = False\n",
    "backend = \"ssd\"\n",
    "\n",
    "for index, r in processedvideos.iterrows():\n",
    "    if forceFaces or pd.isnull(r[\"Faces.file\"]) or not os.path.exists(r[\"Faces.file\"]):\n",
    "        filepath = os.path.join(videos_in, r[\"VideoID\"])\n",
    "        facesdf = getfacedataforvideo(filepath, backend = backend)\n",
    "        if not facesdf is None:\n",
    "            stemname = os.path.splitext(r[\"VideoID\"])[0]\n",
    "            facefile =  os.path.join(data_out, stemname + f\".faces.{backend}.csv\")\n",
    "            facesdf.to_csv(facefile, index=False)\n",
    "            r[\"Faces.file\"] = facefile\n",
    "            r[\"Faces.when\"] = time.strftime(\"%Y-%m-%d %H:%M:%S\", time.gmtime())\n",
    "        else:\n",
    "            r[\"Faces.file\"] = \"\"\n",
    "            r[\"Faces.when\"] = \"\"        \n",
    "        #update this row in processedvideos dataframe\n",
    "        processedvideos.loc[index] = r\n",
    "    else:\n",
    "        print(f\"Already processed {r['VideoID']}\")\n",
    "utils.saveprocessedvideos(processedvideos, data_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "processedvideos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO - add code to match the person labels for the faces to person labels from pose detection (step 1)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "babyjokes",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}