TextDrivenVideoAcceleration_TPAMI_2022.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "440535783a2a4e2f8d8a3a283d45d723",
            "e95c4f533f154e379131fb0121bc5a3c",
            "c8003189cec7445e9098b2a8f390234d",
            "4324cf78079441f4a16b5716054a2496",
            "bc16a9c8b36c44d29a37a37af17263ce",
            "b58a7b8b9ed44d608f28d08a3f395252",
            "65ed4c52833345c78fef56a3a3e1ab23",
            "953d70bcae8143558bcdaacc715ecaa3",
            "afa5cd20beb34476b91e0c9a42d3b643",
            "1acbabb6aa4149b8a2e642c3c95928db",
            "d181716f5e374b898ae8812ae3fde2bd",
            "713ab4d862684b7db9209348da27b979",
            "9b87feb3e90149a392107a2635baab89",
            "a561b10a131d405cb45da3ccfbd34abd",
            "3c178fb4e4b4440a9f97efcb10237cef",
            "77fa013c7f1d4a53804920918b1a6cbe"
          ]
        },
        "id": "m0YnJE2JiDvc",
        "outputId": "36ec6930-b4bc-4700-b18b-ea65e0337119"
      },
      "outputs": [],
      "source": [
        "! pip install yt-dlp\n",
        "! pip install torchvideo\n",
        "\n",
        "import torch\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "\n",
        "model = torch.hub.load('verlab/TextDrivenVideoAcceleration_TPAMI_2022:main', 'TextDrivenVideoAcceleration', pretrained=True, progress=True)\n",
        "model.cuda()\n",
        "model.eval()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K4P4K2LvCKmx",
        "outputId": "bef48aa2-0b37-4369-e959-31ee3dc77ff0"
      },
      "outputs": [],
      "source": [
        "VIDEO_ID = 'jnewhlK2USg'\n",
        "document = ['chop some tuna meat into small pieces',\n",
        "            'mix the tuna pieces and some fish eggs in a bowl',\n",
        "            'add hot sauce and mayo and then mix them together',\n",
        "            'spread some cooked rice on seaweed sushi wrapper',\n",
        "            'add the tuna mix on top and season with some sesame seeds',\n",
        "            'roll it up and press firmly',\n",
        "            'cut the roll into small rolls']\n",
        "\n",
        "! yt-dlp https://www.youtube.com/watch?v=$VIDEO_ID -o $VIDEO_ID --merge-output-format mp4 # Download Video File\n",
        "\n",
        "print('\\nExtracting VDAN+ Features...')\n",
        "semantic_embeddings = model.get_semantic_embeddings(video_filename=f'{VIDEO_ID}.mp4', document=document)\n",
        "\n",
        "print('\\nFast Forwarding Video...')\n",
        "sf = model.fast_forward_video(video_filename=f'{VIDEO_ID}.mp4',\n",
        "                              document=document,\n",
        "                              desired_speedup=12,\n",
        "                              output_video_filename=f'{VIDEO_ID}_FF.avi',\n",
        "                              semantic_embeddings=semantic_embeddings)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "YNFaGWWXIVgM",
        "outputId": "26e91965-bca1-4b05-9814-802d69d8cfd4"
      },
      "outputs": [],
      "source": [
        "! wget -O youcookii_annotations_trainval.tar.gz http://youcook2.eecs.umich.edu/static/YouCookII/youcookii_annotations_trainval.tar.gz\n",
        "! tar -xf youcookii_annotations_trainval.tar.gz\n",
        "! rm youcookii_annotations_trainval.tar.gz\n",
        "\n",
        "import json\n",
        "import cv2\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "annotations = json.load(open('youcookii_annotations_trainval.json'))\n",
        "\n",
        "video_annotations = annotations['database'][VIDEO_ID]['annotations']\n",
        "cmap = plt.cm.get_cmap('hsv', len(video_annotations) + 1)\n",
        "video_fps = cv2.VideoCapture(f'{VIDEO_ID}.mp4').get(5)\n",
        "for idx, region in enumerate(video_annotations):\n",
        "    video_region = region['segment']\n",
        "    plt.axvspan(round(video_region[0] * video_fps), round(video_region[1] * video_fps), alpha=0.5, color=cmap(idx))\n",
        "\n",
        "plt.scatter(sf[:-1], np.array(sf[1:])-np.array(sf[:-1]))\n",
        "plt.title('Agent Skips')\n",
        "plt.xlabel('Frame Number')\n",
        "plt.ylabel('Skip')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from google.colab import files\n",
        "files.download(f'{VIDEO_ID}_FF.avi')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "TextDrivenVideoAcceleration_TPAMI_2022.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1acbabb6aa4149b8a2e642c3c95928db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c178fb4e4b4440a9f97efcb10237cef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4324cf78079441f4a16b5716054a2496": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_953d70bcae8143558bcdaacc715ecaa3",
            "placeholder": "​",
            "style": "IPY_MODEL_65ed4c52833345c78fef56a3a3e1ab23",
            "value": " 97.8M/97.8M [00:00&lt;00:00, 174MB/s]"
          }
        },
        "440535783a2a4e2f8d8a3a283d45d723": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c8003189cec7445e9098b2a8f390234d",
              "IPY_MODEL_4324cf78079441f4a16b5716054a2496"
            ],
            "layout": "IPY_MODEL_e95c4f533f154e379131fb0121bc5a3c"
          }
        },
        "65ed4c52833345c78fef56a3a3e1ab23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "713ab4d862684b7db9209348da27b979": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77fa013c7f1d4a53804920918b1a6cbe",
            "placeholder": "​",
            "style": "IPY_MODEL_3c178fb4e4b4440a9f97efcb10237cef",
            "value": " 824M/824M [00:19&lt;00:00, 45.3MB/s]"
          }
        },
        "77fa013c7f1d4a53804920918b1a6cbe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "953d70bcae8143558bcdaacc715ecaa3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b87feb3e90149a392107a2635baab89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "a561b10a131d405cb45da3ccfbd34abd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "afa5cd20beb34476b91e0c9a42d3b643": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d181716f5e374b898ae8812ae3fde2bd",
              "IPY_MODEL_713ab4d862684b7db9209348da27b979"
            ],
            "layout": "IPY_MODEL_1acbabb6aa4149b8a2e642c3c95928db"
          }
        },
        "b58a7b8b9ed44d608f28d08a3f395252": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bc16a9c8b36c44d29a37a37af17263ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "c8003189cec7445e9098b2a8f390234d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b58a7b8b9ed44d608f28d08a3f395252",
            "max": 102502400,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bc16a9c8b36c44d29a37a37af17263ce",
            "value": 102502400
          }
        },
        "d181716f5e374b898ae8812ae3fde2bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a561b10a131d405cb45da3ccfbd34abd",
            "max": 863810626,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9b87feb3e90149a392107a2635baab89",
            "value": 863810626
          }
        },
        "e95c4f533f154e379131fb0121bc5a3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}