notebooks/2020_0605Detectron2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2020_0605Detectron2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/komazawa-deep-learning/komazawa-deep-learning.github.io/blob/master/notebooks/2020_0605Detectron2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ft0yrLBGBPbk",
        "colab_type": "text"
      },
      "source": [
        "- source: https://colab.research.google.com/drive/1IiMLmS8oUknQJzxiIy72eu0RUV9sBJaH#scrollTo=QHnVupBBn9eR\n",
        "- date: 2020-0605\n",
        "- author: shin asakawa\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QHnVupBBn9eR",
        "colab_type": "text"
      },
      "source": [
        "# Detectron2 Beginner's Tutorial を改変ししました。\n",
        "\n",
        "優れたチュートリアルを提供している フェイスブック人工知能研究所 FAIR に感謝します\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vM54r6jlKTII",
        "colab_type": "text"
      },
      "source": [
        "# detectron2 のインストール"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9_FzH13EjseR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# install dependencies: (use cu101 because colab has CUDA 10.1)\n",
        "!pip install -U torch==1.5 torchvision==0.6 -f https://download.pytorch.org/whl/cu101/torch_stable.html \n",
        "!pip install cython pyyaml==5.1\n",
        "!pip install -U 'git+https://github.com/cocodataset/cocoapi.git#subdirectory=PythonAPI'\n",
        "import torch, torchvision\n",
        "print(torch.__version__, torch.cuda.is_available())\n",
        "!gcc --version\n",
        "# opencv is pre-installed on colab"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b-i4hmGYk1dL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# install detectron2:\n",
        "!pip install detectron2==0.1.3 -f https://dl.fbaipublicfiles.com/detectron2/wheels/cu101/torch1.5/index.html"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZyAvNCJMmvFF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# You may need to restart your runtime prior to this, to let your installation take effect\n",
        "# Some basic setup:\n",
        "# Setup detectron2 logger\n",
        "import detectron2\n",
        "from detectron2.utils.logger import setup_logger\n",
        "setup_logger()\n",
        "\n",
        "# import some common libraries\n",
        "import numpy as np\n",
        "import cv2\n",
        "import random\n",
        "from google.colab.patches import cv2_imshow\n",
        "\n",
        "# import some common detectron2 utilities\n",
        "from detectron2 import model_zoo\n",
        "from detectron2.engine import DefaultPredictor\n",
        "from detectron2.config import get_cfg\n",
        "from detectron2.utils.visualizer import Visualizer\n",
        "from detectron2.data import MetadataCatalog"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vk4gID50K03a",
        "colab_type": "text"
      },
      "source": [
        "# 訓練済 detectron2 モデルの読み込み"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JgKyUL4pngvE",
        "colab_type": "text"
      },
      "source": [
        "COCO データセットから画像を持ってくる\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dq9GY37ml1kr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !wget <URL> -O 保存したいファイル名 という書式です。自習で任意の画像を試すときに参考にしてください\n",
        "!wget http://images.cocodataset.org/val2017/000000439715.jpg -O input.jpg\n",
        "im = cv2.imread(\"./input.jpg\")\n",
        "cv2_imshow(im)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uM1thbN-ntjI",
        "colab_type": "text"
      },
      "source": [
        "<!--Then, we create a detectron2 config and a detectron2 `DefaultPredictor` to run inference on this image.-->\n",
        "画像を認識するために detectron2 の設定と `DefaultPredictor` を作成"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUjkwRsOn1O0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cfg = get_cfg()\n",
        "# add project-specific config (e.g., TensorMask) here if you're not running a model in detectron2's core library\n",
        "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\n",
        "cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model\n",
        "# Find a model from detectron2's model zoo. You can use the https://dl.fbaipublicfiles... url as well\n",
        "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\n",
        "predictor = DefaultPredictor(cfg)\n",
        "outputs = predictor(im)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7d3KxiHO_0gb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# look at the outputs. See https://detectron2.readthedocs.io/tutorials/models.html#model-output-format for specification\n",
        "# 矩形領域 （長方形）の関心領域を表示\n",
        "outputs[\"instances\"].pred_classes\n",
        "outputs[\"instances\"].pred_boxes"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8IRGo8d0qkgR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We can use `Visualizer` to draw the predictions on the image.\n",
        "# 結果を視覚化して表示\n",
        "v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n",
        "v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n",
        "cv2_imshow(v.get_image()[:, :, ::-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qWq1XHfDWiXO",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "\n",
        "# ここから実習\n",
        "\n",
        "以下は実習に関係しますので注意して観察してください。\n",
        "下で自分の画像を持ってきたら，ここから再実行します。\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5d0aJ2zpUg9h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 実体切り分けのデモ\n",
        "cfg = get_cfg()\n",
        "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\"))\n",
        "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml\")\n",
        "predictor = DefaultPredictor(cfg)\n",
        "outputs = predictor(im)\n",
        "v = Visualizer(im[:,:,::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n",
        "v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n",
        "cv2_imshow(v.get_image()[:, :, ::-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "roTj1N9F5uJ5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 汎視覚的分割のデモ\n",
        "cfg = get_cfg()\n",
        "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-PanopticSegmentation/panoptic_fpn_R_101_3x.yaml\"))\n",
        "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-PanopticSegmentation/panoptic_fpn_R_101_3x.yaml\")\n",
        "predictor = DefaultPredictor(cfg)\n",
        "panoptic_seg, segments_info = predictor(im)[\"panoptic_seg\"]\n",
        "v = Visualizer(im[:, :, ::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n",
        "v = v.draw_panoptic_seg_predictions(panoptic_seg.to(\"cpu\"), segments_info)\n",
        "cv2_imshow(v.get_image()[:, :, ::-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "herJHqthTkSu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# キーポイント検出のデモ\n",
        "cfg = get_cfg()\n",
        "cfg.merge_from_file(model_zoo.get_config_file(\"COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml\"))\n",
        "cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7  # set threshold for this model\n",
        "cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(\"COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x.yaml\")\n",
        "predictor = DefaultPredictor(cfg)\n",
        "outputs = predictor(im)\n",
        "v = Visualizer(im[:,:,::-1], MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), scale=1.2)\n",
        "v = v.draw_instance_predictions(outputs[\"instances\"].to(\"cpu\"))\n",
        "cv2_imshow(v.get_image()[:, :, ::-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rtA6kS93-tPF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 任意の画像をインターネット上から検索して，次の行に貼り付けます。\n",
        "!wget https://github.com/komazawa-deep-learning/komazawa-deep-learning.github.io/raw/master/assets/1996Jerusalem_wall_of_tears.png -O input.png\n",
        "im = cv2.imread(\"./input.png\")\n",
        "cv2_imshow(im)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cyA4VmKcF61k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 動画の作成，時間がかかります\n",
        "# Run frame-by-frame inference demo on this video (takes 3-4 minutes)\n",
        "# Using a model trained on COCO dataset\n",
        "!git clone https://github.com/facebookresearch/detectron2\n",
        "!python detectron2/demo/demo.py --config-file detectron2/configs/COCO-PanopticSegmentation/panoptic_fpn_R_101_3x.yaml --video-input video-clip.mp4 --confidence-threshold 0.6 --output video-output.mkv \\\n",
        "  --opts MODEL.WEIGHTS detectron2://COCO-PanopticSegmentation/panoptic_fpn_R_101_3x/139514519/model_final_cafdb1.pkl"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OpLg_MAQGPUT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Download the results\n",
        "from google.colab import files\n",
        "files.download('video-output.mkv')"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}