notebooks/egoexo/EgoExo_Expert_Commentary_Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d6f01e75-0642-4b63-98f3-2d4c0564198c",
   "metadata": {},
   "source": [
    "# Introduction\n",
    "\n",
    "Expert Commentary was an annotation task annotated by ~50 experts across a large portion of the dataset. Some takes are annotated multiple times by different experts. \n",
    "\n",
    "The annotation task required experts to provide:\n",
    "- Audio commentary at timepoints (of their choosing) with optional drawing overlays\n",
    "- Profiency score (and reason why in plain-text) of the performance of the camera wearer/participant\n",
    "\n",
    "For audio commentary, we have transcribed each commentary with Whisper (large-v2). Here is how you use the annotations, with some basic analysis and visualization (how to use the drawing data)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1372450-8f9a-4a1e-b29f-4c6088b5cce1",
   "metadata": {},
   "source": [
    "## Data Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98e6edd-5ce9-467f-a72e-b32bd0dc190b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import random\n",
    "from collections import defaultdict\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import spacy\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f47478c0-2e37-49f0-bd28-bb69128c9bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "egoexo_root = \"/dataset/placeholder/dir/\" # NOTE: changeme to your download path\n",
    "# egoexo_root = \"/large_experiments/egoexo/v2/\"\n",
    "egoexo_annotation_root = os.path.join(egoexo_root, \"annotations\")\n",
    "assert os.path.exists(egoexo_root), \"please make sure you have downloaded egoexo or check your path\"\n",
    "assert os.path.exists(egoexo_annotation_root), \"please download annotations with --parts annotations or check your path\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4496e4fe-f40b-4219-842b-b44d345b51df",
   "metadata": {},
   "outputs": [],
   "source": [
    "takes = json.load(open(os.path.join(egoexo_root, \"takes.json\")))\n",
    "takes_by_name = {t[\"take_name\"]: t for t in takes}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf7e3d76-c6a1-4827-9ce1-7022aaa16a09",
   "metadata": {},
   "outputs": [],
   "source": [
    "expert_commentary_transc_path = os.path.join(egoexo_annotation_root, \"expert_commentary_train.json\")\n",
    "assert os.path.exists(expert_commentary_transc_path), \"please re-download egoexo's annotations (--parts annotations) to get expert commentary\"\n",
    "\n",
    "ecs = json.load(open(expert_commentary_transc_path))\n",
    "all_anns = ecs[\"annotations\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02b61791-7eca-47ba-824e-6820de8b4669",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_transc = []\n",
    "for take_uid, anns in all_anns.items():\n",
    "    for ann in anns:\n",
    "        all_transc.extend(\n",
    "            [\n",
    "                {\n",
    "                    \"take\": ann[\"take_name\"],\n",
    "                    \"commentary\": ann[\"commentary\"],\n",
    "                    **x,\n",
    "                }\n",
    "                for x in ann[\"commentary_data\"]\n",
    "            ]\n",
    "        )\n",
    "len(all_transc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "587411ce-2eaa-4b26-b2b0-0ad53f95a87c",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = [x for x in all_transc if x[\"error\"]]\n",
    "all_transc_succ = [x for x in all_transc if not x[\"error\"]]\n",
    "len(all_transc), len(all_transc_succ), len(errors)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50d3a6ba-63d5-402c-acac-e9332c011561",
   "metadata": {},
   "source": [
    "## Basic Usage & Analysis on Transcriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3784c221-c0f3-4b2a-afcd-416d19f251ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "run_spacy_analysis = False # set me to True, be warned this will take ~20minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8375c14c-6ca7-49b0-959b-1fc9f9e63e3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_md\")\n",
    "stats = {\n",
    "    \"num_nouns\": [],\n",
    "    \"num_verbs\": [],\n",
    "    \"num_sents\": [],\n",
    "    \"num_words\": [],\n",
    "    \"words_per_sentence\": [],\n",
    "}\n",
    "\n",
    "noun_counts = defaultdict(int)\n",
    "verb_counts = defaultdict(int)\n",
    "if run_spacy_analysis:\n",
    "    for x in tqdm(all_transc_succ):\n",
    "        doc = nlp(x[\"text\"])\n",
    "        num_sents = len(list(doc.sents))\n",
    "        num_words = len(doc)\n",
    "        words_per_sentence = num_words / num_sents if num_sents > 0 else None\n",
    "        toks_by_class = defaultdict(list)\n",
    "        for tok in doc:\n",
    "            toks_by_class[tok.pos_].append(tok)\n",
    "        num_nouns = len(toks_by_class[\"NOUN\"]) + len(toks_by_class[\"PROPN\"])\n",
    "        num_verbs = len(toks_by_class[\"VERBS\"])\n",
    "        for tok in toks_by_class[\"NOUN\"]:\n",
    "            noun_counts[tok.text] += 1\n",
    "        for tok in toks_by_class[\"PROPN\"]:\n",
    "            noun_counts[tok.text] += 1\n",
    "        for tok in toks_by_class[\"VERB\"]:\n",
    "            if tok.text == \"'s\":\n",
    "                continue\n",
    "            verb_counts[tok.text] += 1\n",
    "    \n",
    "        stats[\"num_nouns\"].append(num_nouns)\n",
    "        stats[\"num_verbs\"].append(num_verbs)\n",
    "        stats[\"num_sents\"].append(num_sents)\n",
    "        stats[\"num_words\"].append(num_words)\n",
    "        stats[\"words_per_sentence\"].append(words_per_sentence)\n",
    "\n",
    "noun_counts_sorted = sorted(noun_counts.items(), key=lambda x: -x[1])\n",
    "verb_counts_sorted = sorted(verb_counts.items(), key=lambda x: -x[1])\n",
    "\n",
    "num_anns = len({x[\"commentary\"] for x in all_transc_succ})\n",
    "num_takes = len({x[\"take\"] for x in all_transc_succ})\n",
    "\n",
    "comms_per_ann = defaultdict(list)\n",
    "for x in all_transc_succ:\n",
    "    comms_per_ann[x[\"commentary\"]].append(x)\n",
    "\n",
    "comms_per_ann_arr = np.array([len(xs) for xs in comms_per_ann.values()])\n",
    "\n",
    "comm_per_min = []\n",
    "for comm, xs in comms_per_ann.items():\n",
    "    tn = xs[0][\"take\"]\n",
    "    if tn not in takes_by_name:\n",
    "        continue\n",
    "    take = takes_by_name[tn]\n",
    "    take_min = take[\"duration_sec\"] / 60\n",
    "    num_comms = len(xs)\n",
    "    comm_per_min.append(num_comms / take_min)\n",
    "\n",
    "comms_per_min = np.array(comm_per_min)\n",
    "\n",
    "stats_df = pd.DataFrame(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e9bad188",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_experts = len(set(ann[\"commentary\"].split(\"/\")[-1] for anns in all_anns.values() for ann in anns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2fc4c1f-332f-4345-a9b7-5739c038fdf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\n",
    "f\"\"\"\n",
    "# Annotations = {num_anns}\n",
    "# Takes Annotated = {num_takes}\n",
    "# Commentaries = {len(all_transc_succ)}\n",
    "Avg Commentaries per Annotation = {comms_per_ann_arr.mean():.3f} (std dev = {comms_per_ann_arr.std():.3f})\n",
    "# Sentences = {stats_df.num_sents.sum()}\n",
    "Avg Sentences per Commentary = {stats_df.num_sents.mean():.3f} (std dev = {stats_df.num_sents.std():.3f})\n",
    "# Words = {stats_df.num_words.sum()}\n",
    "Avg Words per Sentence = {stats_df.words_per_sentence.mean():.3f} (std dev = {stats_df.words_per_sentence.std():.3f})\n",
    "# Unique Nouns = {len(noun_counts_sorted)}\n",
    "# Unique Verbs = {len(verb_counts_sorted)}\n",
    "Average Commentaries per Minute = {comms_per_min.mean():.3f}\n",
    "# Experts = {num_experts}\n",
    "\"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b1c2d5c-123a-4298-9dce-582deaf9a9d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for x, count in noun_counts_sorted[0:150]:\n",
    "    print(f\"{x} : {count}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52cfb3d7-077b-4f21-a6c6-44f3b839b969",
   "metadata": {},
   "outputs": [],
   "source": [
    "for x, count in verb_counts_sorted[0:150]:\n",
    "    print(f\"{x} : {count}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c67b296d-845b-4a2c-8091-f82c1a2c18f5",
   "metadata": {},
   "source": [
    "## Profiency Score & Path Drawing for an Annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a783b48-c014-4e80-a265-b24619913305",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ego4d.egoexo.expert_commentary import get_paths_for_commentary_time\n",
    "\n",
    "needs_visualization = True # do you want to visualize the path drawing?\n",
    "if needs_visualization:\n",
    "    from ego4d.research.readers import TorchAudioStreamReader\n",
    "    import cv2\n",
    "    from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ead5412-50ef-4b59-860e-65ae5cba9c87",
   "metadata": {},
   "outputs": [],
   "source": [
    "takes_with_comm = list(all_anns.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d7b0932-41fc-458b-99a7-1d5e92cc6345",
   "metadata": {},
   "outputs": [],
   "source": [
    "takes_with_vid = {\n",
    "    take[\"take_uid\"]\n",
    "    for take in takes\n",
    "    if take[\"frame_aligned_videos\"][\"best_exo\"][\"0\"][\"relative_path\"] is not None\n",
    "}\n",
    "takes_to_sample = set(takes_with_comm) & set(takes_with_vid)\n",
    "take_uid = random.sample(takes_with_comm, k=1)[0]\n",
    "annotator_idx = random.randint(0, len(all_anns[take_uid]) - 1)\n",
    "\n",
    "# here are example (take, annotator) pairs where there are cleared out paths\n",
    "# take_uid, annotator_idx = ('6d258ba3-363e-4a40-b739-2b1b6e13fa8a', 1)\n",
    "# take_uid, annotator_idx = ('3043fd07-a52a-4adc-9a19-12e7e1c29df4', 2)\n",
    "\n",
    "ann = all_anns[take_uid][annotator_idx]\n",
    "annotator_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dec53ee-c67b-4f2c-8189-34c460f576ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data_path = os.path.join(egoexo_annotation_root, \"expert_commentary/\", ann[\"commentary\"], \"data.json\")\n",
    "data_path = os.path.join(\"/checkpoint/miguelmartin/expert_commentary/exports/240207/data/\", ann[\"commentary\"], \"data.json\")\n",
    "data = json.load(open(data_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d06d39b8-05ae-42ea-9f16-ef36d81f2b5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rating of the performance (why & a 1-10 score)\n",
    "data[\"proficiency\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4eaf4b07-ab21-4f7f-a1cf-182b1b6e0ece",
   "metadata": {},
   "outputs": [],
   "source": [
    "take = takes_by_name[ann[\"take_name\"]]\n",
    "best_exo_video_path = os.path.join(\n",
    "    egoexo_root,\n",
    "    take[\"root_dir\"],\n",
    "    take[\"frame_aligned_videos\"][\"best_exo\"][\"0\"][\"relative_path\"],\n",
    ")\n",
    "assert os.path.exists(best_exo_video_path), f\"\"\"\n",
    "please download collages for this take (via `--parts take`), use `--uid {take['take_uid']}` to just download this take\n",
    "\"\"\"\n",
    "best_exo_video_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86b2f2da-c130-4dbd-9e76-8433e1b25667",
   "metadata": {},
   "outputs": [],
   "source": [
    "video_reader = TorchAudioStreamReader(\n",
    "    best_exo_video_path,\n",
    "    resize=None,\n",
    "    crop=None,\n",
    "    mean=None,\n",
    "    std=None,\n",
    "    frame_window_size=1,\n",
    "    stride=1,\n",
    "    gpu_idx=-1,\n",
    "    axis_order=\"thwc\",\n",
    "    uint8_scale=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2f86e05-f702-4a8f-9265-43d184215038",
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_paths(paths, img):\n",
    "    ret = img.copy()\n",
    "    h, w, c = img.shape\n",
    "    round_fn = int # TODO: could round to closest integer\n",
    "    for path in paths:\n",
    "        # NOTE: points are scaled \n",
    "        from_pt = (round_fn(path[\"from\"][\"x\"] * w), round_fn(path[\"from\"][\"y\"] * h), )\n",
    "        to_pt = (round_fn(path[\"to\"][\"x\"] * w), round_fn(path[\"to\"][\"y\"] * h), )\n",
    "        ret = cv2.line(\n",
    "            ret,\n",
    "            from_pt,\n",
    "            to_pt,\n",
    "            (255, 0, 0),\n",
    "            2,\n",
    "        )\n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15d60bac-ef85-40ad-9f97-8fef6fb20397",
   "metadata": {},
   "outputs": [],
   "source": [
    "comm_idx = None\n",
    "if comm_idx is None:\n",
    "    for comm_idx, comm in enumerate(data[\"annotations\"]):\n",
    "        if len(comm[\"events\"]) > 0:\n",
    "            break\n",
    "else:\n",
    "    comm = data[\"annotations\"][comm_idx]\n",
    "if len(comm[\"events\"]) == 0:\n",
    "    print(\"WARN: no draw events associated to this commentary\")\n",
    "\n",
    "comm_dur = comm[\"duration_approx\"]\n",
    "\n",
    "# sample `num_pts` points from [0, comm_dur) uniformally \n",
    "num_pts = 9\n",
    "comm_ts = np.linspace(0, comm_dur, num_pts)\n",
    "# get the path for each timestamp t\n",
    "comm_paths_per_t = [get_paths_for_commentary_time(comm, t) for t in comm_ts]\n",
    "\n",
    "# compute the number of points per timestamp\n",
    "comm_num_paths_per_t = [len(xs) for xs in comm_paths_per_t]\n",
    "\n",
    "dict(zip(comm_ts, comm_num_paths_per_t)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e996b1f-407b-4674-a7e5-cc78a7d9f70a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# render the paths for the frame\n",
    "comm_video_t = comm[\"video_time\"]\n",
    "frame = video_reader[int(comm_video_t * 30)][\"video\"][0]\n",
    "img = frame.numpy()\n",
    "img_per_t = [\n",
    "    Image.fromarray(draw_paths(paths, img))\n",
    "    for paths in comm_paths_per_t\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d6490e2-7d03-472b-a170-ded85d83b487",
   "metadata": {},
   "outputs": [],
   "source": [
    "img_per_t[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4aa29f0a-96b2-4476-bde3-33abce9873a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "img_per_t[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d482e8d-5332-407e-a1d2-9e7421189402",
   "metadata": {},
   "outputs": [],
   "source": [
    "img_per_t[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf480fcd-1e68-4c24-9ae4-4837a9ac9b43",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "expert_commentary",
   "language": "python",
   "name": "expert_commentary"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}