notebooks/manage_video_batch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c14e45cf",
   "metadata": {},
   "source": [
    "# Managing a local MegaDetector video batch\n",
    "\n",
    "This notebook represents an interactive process for running MegaDetector on large batches of videos, including typical and optional postprocessing steps.\n",
    "\n",
    "This notebook is auto-generated from manage_video_batch.py (a cell-delimited .py file that is used the same way, typically in Spyder or VS Code).\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3766ee0",
   "metadata": {},
   "source": [
    "## Imports and constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23c9fd9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "from megadetector.utils import path_utils\n",
    "from megadetector.detection import video_utils\n",
    "\n",
    "input_folder = '/datadrive/data'\n",
    "frame_folder_base = '/datadrive/frames'\n",
    "\n",
    "assert os.path.isdir(input_folder)\n",
    "os.makedirs(frame_folder_base,exist_ok=True)\n",
    "\n",
    "quality = 90\n",
    "max_width = 1600\n",
    "every_n_frames = 10\n",
    "recursive = True\n",
    "overwrite = True\n",
    "parallelization_uses_threads = True\n",
    "n_workers = 8"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "540118f0",
   "metadata": {},
   "source": [
    "## Split videos into frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22689d3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert os.path.isdir(input_folder)\n",
    "os.makedirs(frame_folder_base,exist_ok=True)\n",
    "\n",
    "frame_filenames_by_video,fs_by_video,video_filenames = \\\n",
    "    video_utils.video_folder_to_frames(input_folder=input_folder,\n",
    "                                       output_folder_base=frame_folder_base,\n",
    "                                       recursive=recursive,\n",
    "                                       overwrite=overwrite,\n",
    "                                       n_threads=n_workers,\n",
    "                                       every_n_frames=every_n_frames,\n",
    "                                       parallelization_uses_threads=parallelization_uses_threads,\n",
    "                                       quality=quality,\n",
    "                                       max_width=max_width)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2d1765d",
   "metadata": {},
   "source": [
    "## List frame files, break into folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd7ddb13",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Each leaf-node folder *should* correspond to a video; we're going to verify that below.\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "frame_files = path_utils.find_images(frame_folder_base,True)\n",
    "frame_files = [s.replace('\\\\','/') for s in frame_files]\n",
    "print('Enumerated {} total frames'.format(len(frame_files)))\n",
    "\n",
    "# Find unique (relative) folders\n",
    "folder_to_frame_files = defaultdict(list)\n",
    "\n",
    "# fn = frame_files[0]\n",
    "for fn in frame_files:\n",
    "    folder_name = os.path.dirname(fn)\n",
    "    folder_name = os.path.relpath(folder_name,frame_folder_base)\n",
    "    folder_to_frame_files[folder_name].append(fn)\n",
    "\n",
    "print('Found {} folders for {} files'.format(len(folder_to_frame_files),len(frame_files)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73f998f9",
   "metadata": {},
   "source": [
    "## List videos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb4051e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "video_filenames = video_utils.find_videos(input_folder,recursive=True)\n",
    "video_filenames = [os.path.relpath(fn,input_folder) for fn in video_filenames]\n",
    "print('Input folder contains {} videos'.format(len(video_filenames)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d6c656a",
   "metadata": {},
   "source": [
    "## Check for videos that don't have corresponding frame folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae74c9b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# These are almost always corrupt videos; if you have a million camera trap videos,\n",
    "# you will inevitably have a few videos that completely failed to open.  If *all* of\n",
    "# your videos failed to open, something is up, but if it's a small percentage, move right\n",
    "# along.\n",
    "\n",
    "# list(folder_to_frame_files.keys())[0]\n",
    "# video_filenames[0]\n",
    "\n",
    "missing_videos = []\n",
    "\n",
    "# fn = video_filenames[0]\n",
    "for relative_fn in video_filenames:\n",
    "    if relative_fn not in folder_to_frame_files:\n",
    "        missing_videos.append(relative_fn)\n",
    "\n",
    "print('{} of {} folders are missing frames entirely'.format(len(missing_videos),\n",
    "                                                            len(video_filenames)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ab07bef",
   "metadata": {},
   "source": [
    "## Check for videos with very few frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5359b1b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Same as above; if you have a million camera trap videos, a few will inevitably be\n",
    "# corrupted after a few frames.  This cell checks for videos that have some frames,\n",
    "# but not *all* the frames.  It should be a small number, but if you have a huge\n",
    "# dataset, it won't be zero.  If you can live with this number, move right along.\n",
    "\n",
    "min_frames_for_valid_video = 10\n",
    "\n",
    "low_frame_videos = []\n",
    "\n",
    "for folder_name in folder_to_frame_files.keys():\n",
    "    frame_files = folder_to_frame_files[folder_name]\n",
    "    if len(frame_files) < min_frames_for_valid_video:\n",
    "        low_frame_videos.append(folder_name)\n",
    "\n",
    "print('{} of {} folders have fewer than {} frames'.format(\n",
    "    len(low_frame_videos),len(video_filenames),min_frames_for_valid_video))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "544c9230",
   "metadata": {},
   "source": [
    "## Print the list of videos that are problematic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "226ed54a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Videos that could not be decoded:\\n')\n",
    "\n",
    "for fn in missing_videos:\n",
    "    print(fn)\n",
    "\n",
    "print('\\nVideos with fewer than {} decoded frames:\\n'.format(min_frames_for_valid_video))\n",
    "\n",
    "for fn in low_frame_videos:\n",
    "    print(fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d44d9d7",
   "metadata": {},
   "source": [
    "## Process images like we would for any other camera trap job"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81b049ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ...typically using manage_local_batch.py or manage_local_batch.ipynb, but do this however\n",
    "# you like, as long as you get a results file at the end.\n",
    "#\n",
    "# If you do RDE, remember to use the second folder from the bottom, rather than the\n",
    "# bottom-most folder."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d480df26",
   "metadata": {},
   "source": [
    "## Convert frame results to video results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d920d52",
   "metadata": {},
   "outputs": [],
   "source": [
    "from megadetector.detection.video_utils import frame_results_to_video_results\n",
    "\n",
    "filtered_output_filename = '/results/organization/stuff.json'\n",
    "video_output_filename = filtered_output_filename.replace('.json','_aggregated.json')\n",
    "frame_results_to_video_results(filtered_output_filename,video_output_filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ac74830",
   "metadata": {},
   "source": [
    "## Confirm that the videos in the .json file are what we expect them to be"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5a8fd24",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(video_output_filename,'r') as f:\n",
    "    video_results = json.load(f)\n",
    "\n",
    "video_filenames_set = set(video_filenames)\n",
    "\n",
    "filenames_in_video_results_set = set([im['file'] for im in video_results['images']])\n",
    "\n",
    "for fn in filenames_in_video_results_set:\n",
    "    assert fn in video_filenames_set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93effc61",
   "metadata": {},
   "source": [
    "## Scrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d7c12cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "if False:\n",
    "\n",
    "    pass\n",
    "\n",
    "    #%% Render all detections to videos\n",
    "\n",
    "    from megadetector.visualization.visualize_detector_output import visualize_detector_output\n",
    "    from megadetector.utils.path_utils import insert_before_extension\n",
    "    from megadetector.detection.video_utils import frames_to_video\n",
    "\n",
    "    rendering_confidence_threshold = 0.1\n",
    "    target_fs = 100\n",
    "    fourcc = None\n",
    "\n",
    "    # Render detections to images\n",
    "    frame_rendering_output_dir = os.path.expanduser('~/tmp/rendered-frames')\n",
    "    os.makedirs(frame_rendering_output_dir,exist_ok=True)\n",
    "\n",
    "    video_rendering_output_dir = os.path.expanduser('~/tmp/rendered-videos')\n",
    "    os.makedirs(video_rendering_output_dir,exist_ok=True)\n",
    "\n",
    "    frames_json = filtered_output_filename\n",
    "\n",
    "    detected_frame_files = visualize_detector_output(\n",
    "        detector_output_path=frames_json,\n",
    "        out_dir=frame_rendering_output_dir,\n",
    "        images_dir=frame_folder_base,\n",
    "        confidence_threshold=rendering_confidence_threshold,\n",
    "        preserve_path_structure=True,\n",
    "        output_image_width=-1)\n",
    "\n",
    "    detected_frame_files = [s.replace('\\\\','/') for s in detected_frame_files]\n",
    "\n",
    "    output_video_folder = os.path.expanduser('~/tmp/rendered-videos')\n",
    "    os.makedirs(output_video_folder,exist_ok=True)\n",
    "\n",
    "    # i_video=0; input_video_file_abs = video_filenames[i_video]\n",
    "    for i_video,input_video_file_abs in enumerate(video_filenames):\n",
    "\n",
    "        video_fs = fs_by_video[i_video]\n",
    "        rendering_fs = target_fs / every_n_frames\n",
    "\n",
    "        input_video_file_relative = os.path.relpath(input_video_file_abs,input_folder)\n",
    "        video_frame_output_folder = os.path.join(frame_rendering_output_dir,input_video_file_relative)\n",
    "        video_frame_output_folder = video_frame_output_folder.replace('\\\\','/')\n",
    "        assert os.path.isdir(video_frame_output_folder), \\\n",
    "            'Could not find frame folder for video {}'.format(input_video_file_relative)\n",
    "\n",
    "        # Find the corresponding rendered frame folder\n",
    "        video_frame_files = [fn for fn in detected_frame_files if \\\n",
    "                             fn.startswith(video_frame_output_folder)]\n",
    "        assert len(video_frame_files) > 0, 'Could not find rendered frames for video {}'.format(\n",
    "            input_video_file_relative)\n",
    "\n",
    "        # Select the output filename for the rendered video\n",
    "        if input_folder == video_rendering_output_dir:\n",
    "            video_output_file = insert_before_extension(input_video_file_abs,'annotated','_')\n",
    "        else:\n",
    "            video_output_file = os.path.join(video_rendering_output_dir,input_video_file_relative)\n",
    "\n",
    "        os.makedirs(os.path.dirname(video_output_file),exist_ok=True)\n",
    "\n",
    "        # Create the output video\n",
    "        print('Rendering detections for video {} to {} at {} fps (original video {} fps)'.format(\n",
    "            input_video_file_relative,video_output_file,rendering_fs,video_fs))\n",
    "\n",
    "        frames_to_video(video_frame_files,\n",
    "                        rendering_fs,\n",
    "                        video_output_file,\n",
    "                        codec_spec='mp4v')\n",
    "\n",
    "    # ...for each video\n",
    "\n",
    "\n",
    "    #%% Render one or more sample videos...\n",
    "\n",
    "    # ...while we still have the frames and detections around\n",
    "\n",
    "    ## Imports\n",
    "\n",
    "    from megadetector.visualization import visualize_detector_output\n",
    "    from megadetector.detection.video_utils import frames_to_video\n",
    "\n",
    "\n",
    "    ## Constants and paths\n",
    "\n",
    "    confidence_threshold = 0.2\n",
    "    input_fs = 30\n",
    "\n",
    "    filtered_output_filename = '/a/b/c/blah_detections.filtered_rde_0.150_0.850_10_1.000.json'\n",
    "    video_fn_relative = '4.10cam6/IMG_0022.MP4'\n",
    "    output_video_base = os.path.expanduser('~/tmp/video_preview')\n",
    "\n",
    "\n",
    "    ## Filename handling\n",
    "\n",
    "    video_fn_relative = video_fn_relative.replace('\\\\','/')\n",
    "    video_fn_flat = video_fn_relative.replace('/','#')\n",
    "    video_name = os.path.splitext(video_fn_flat)[0]\n",
    "    output_video = os.path.join(output_video_base,'{}_detections.mp4'.format(video_name))\n",
    "    output_fs = input_fs / every_n_frames\n",
    "\n",
    "    rendered_detections_folder = os.path.join(output_video_base,'rendered_detections_{}'.format(video_name))\n",
    "    os.makedirs(rendered_detections_folder,exist_ok=True)\n",
    "\n",
    "\n",
    "    ## Find frames corresponding to this video\n",
    "\n",
    "    with open(filtered_output_filename,'r') as f:\n",
    "        frame_results = json.load(f)\n",
    "\n",
    "    frame_results_this_video = []\n",
    "\n",
    "    # im = frame_results['images'][0]\n",
    "    for im in frame_results['images']:\n",
    "        if im['file'].replace('\\\\','/').startswith(video_fn_relative):\n",
    "            frame_results_this_video.append(im)\n",
    "\n",
    "    assert len(frame_results_this_video) > 0, \\\n",
    "        'No frame results matched {}'.format(video_fn_relative)\n",
    "    print('Found {} matching frame results'.format(len(frame_results_this_video)))\n",
    "\n",
    "    frame_results['images'] = frame_results_this_video\n",
    "\n",
    "    frames_json = os.path.join(rendered_detections_folder,video_fn_flat + '.json')\n",
    "\n",
    "    with open(frames_json,'w') as f:\n",
    "        json.dump(frame_results,f,indent=1)\n",
    "\n",
    "\n",
    "    ## Render detections on those frames\n",
    "\n",
    "    detected_frame_files = visualize_detector_output.visualize_detector_output(\n",
    "        detector_output_path=frames_json,\n",
    "        out_dir=rendered_detections_folder,\n",
    "        images_dir=frame_folder_base,\n",
    "        confidence_threshold=confidence_threshold,\n",
    "        preserve_path_structure=True,\n",
    "        output_image_width=-1)\n",
    "\n",
    "\n",
    "    ## Render the output video\n",
    "\n",
    "    frames_to_video(detected_frame_files, output_fs, output_video, codec_spec='h264')\n",
    "\n",
    "    # from megadetector.utils.path_utils import open_file; open_file(output_video)\n",
    "\n",
    "\n",
    "    #%% Test a possibly-broken video\n",
    "\n",
    "    fn = '/datadrive/tmp/video.AVI'\n",
    "\n",
    "    fs = video_utils.get_video_fs(fn)\n",
    "    print(fs)\n",
    "\n",
    "    tmpfolder = '/home/user/tmp/frametmp'\n",
    "    os.makedirs(tmpfolder,exist_ok=True)\n",
    "\n",
    "    video_utils.video_to_frames(fn, tmpfolder, verbose=True, every_n_frames=10)\n",
    "\n",
    "\n",
    "    #%% List videos in a folder\n",
    "\n",
    "    input_folder = '/datadrive/tmp/organization/data'\n",
    "    video_filenames = video_utils.find_videos(input_folder,recursive=True)\n",
    "\n",
    "\n",
    "    #%% Estimate the extracted size of a folder by sampling a few videos\n",
    "\n",
    "    n_videos_to_sample = 5\n",
    "\n",
    "    video_filenames = video_utils.find_videos(input_folder,recursive=True)\n",
    "    import random\n",
    "    random.seed(0)\n",
    "    sampled_videos = random.sample(video_filenames,n_videos_to_sample)\n",
    "    assert len(sampled_videos) == n_videos_to_sample\n",
    "\n",
    "    size_test_frame_folder = os.path.join(frame_folder_base,'size-test')\n",
    "    if quality is not None:\n",
    "        size_test_frame_folder += '_' + str(quality)\n",
    "    os.makedirs(size_test_frame_folder,exist_ok=True)\n",
    "\n",
    "    total_input_size = 0\n",
    "    total_output_size = 0\n",
    "\n",
    "    # i_video = 0; video_fn = sampled_videos[i_video]\n",
    "    for i_video,video_fn in enumerate(sampled_videos):\n",
    "\n",
    "        print('Processing video {}'.format(video_fn))\n",
    "        frame_output_folder_this_video = os.path.join(size_test_frame_folder,\n",
    "                                                      'video_{}'.format(str(i_video).zfill(4)))\n",
    "        os.makedirs(frame_output_folder_this_video,exist_ok=True)\n",
    "        video_utils.video_to_frames(video_fn,\n",
    "                                    frame_output_folder_this_video,\n",
    "                                    verbose=True,\n",
    "                                    every_n_frames=every_n_frames,\n",
    "                                    quality=quality,\n",
    "                                    max_width=max_width)\n",
    "\n",
    "        from megadetector.utils.path_utils import _get_file_size,get_file_sizes\n",
    "        video_size =_get_file_size(video_fn)[1]\n",
    "        assert video_size > 0\n",
    "        total_input_size += video_size\n",
    "\n",
    "        frame_size = get_file_sizes(frame_output_folder_this_video)\n",
    "        frame_size = sum(frame_size.values())\n",
    "        assert frame_size > 0\n",
    "        total_output_size += frame_size\n",
    "\n",
    "    import shutil # noqa\n",
    "    # shutil.rmtree(size_test_frame_folder)\n",
    "    import humanfriendly\n",
    "    print('')\n",
    "    print('Video size: {}'.format(humanfriendly.format_size(total_input_size)))\n",
    "    print('Frame size: {}'.format(humanfriendly.format_size(total_output_size)))\n",
    "    print('Ratio: {}'.format(total_output_size/total_input_size))"
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 5
}