RBNR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RBNR",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "mount_file_id": "1EgNh4Wx4Ld_mdXhgoDPzXCac3njprUup",
      "authorship_tag": "ABX9TyNJlW0z/0VnTZpvZeLwQmqk",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/leoagneau/Bib_Racer/blob/master/RBNR.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kBm50BCNW9jF",
        "colab_type": "text"
      },
      "source": [
        "## Prepare drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBf1xdsmcuXr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b8540f06-18ab-4504-a247-67784b43a49a"
      },
      "source": [
        "# Mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "# Set data path\n",
        "#root_path = '/content/drive/My Drive/'\n",
        "#proj_path = root_path + 'ML_Projects/Bib_Racer/'\n",
        "!ln -s /content/drive/My\\ Drive/ML_Projects/Bib_Racer /content/bib_racer\n",
        "proj_path = '/content/bib_racer/'\n",
        "data_path = proj_path + 'data/RBNR_data/datasets/'"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l5VoSwB9XExD",
        "colab_type": "text"
      },
      "source": [
        "## Load RNBR data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lluoXuOe-FN7",
        "colab_type": "text"
      },
      "source": [
        "### Load & resize images"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qaP9qC01-Tqv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import cv2\n",
        "from google.colab.patches import cv2_imshow  # Colab only, to replace cv2.imshow\n",
        "img_dims={}\n",
        "os.chdir(data_path)\n",
        "orig_img_path = data_path + 'valid/'\n",
        "yolo_img_path = data_path + 'yolo_valid/'\n",
        "yolo_w = 416\n",
        "yolo_h = 416\n",
        "for file in os.scandir(orig_img_path):\n",
        "  if (file.path.endswith(\".JPG\")):\n",
        "    img = cv2.imread(file.path, cv2.IMREAD_UNCHANGED)\n",
        "    img_dims[file.name.split(\".\")[0]] = [img.shape[1], img.shape[0]]  #[width, height]\n",
        "    # Resize the image\n",
        "    cv2.imwrite(yolo_img_path+file.name, cv2.resize(img, (yolo_w, yolo_h), interpolation=cv2.INTER_AREA))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bhaH2xXZJmep",
        "colab_type": "text"
      },
      "source": [
        "### Load and process mat file, convert to Yolo txt file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L0uM5Mb_KA9X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "from scipy.io import loadmat\n",
        "lb_path = data_path + 'valid_labels/'\n",
        "yolo_lb_path = data_path + 'yolo_train_labels/'\n",
        "for file in os.scandir(lb_path):\n",
        "  if (file.path.endswith(\".mat\")):\n",
        "    m=loadmat(file)\n",
        "    key = file.name.split(\".\")[0]\n",
        "    print(key+':', img_dims[key])\n",
        "    with open(yolo_lb_path+'/'+key+'.txt', 'w') as yolo_f:\n",
        "      for f in m['facep']:\n",
        "        f=f.astype(np.float, copy=False)\n",
        "        xcenter = (f[3]+f[2])/(2*img_dims[key][0])  # (x1+x2)/(2*width)\n",
        "        ycenter = (f[0]+f[1])/(2*img_dims[key][1])  # (y1+y2)/(2*height)\n",
        "        width = (f[3]-f[2])/img_dims[key][0]  # (x2-x1)/(2*width)\n",
        "        height = (f[1]-f[0])/img_dims[key][1]  # (y2-y1)/(2*height)\n",
        "        print(0, xcenter, ycenter, width, height, file=yolo_f)\n",
        "      for t in m['tagp']:\n",
        "        t=t.astype(np.float, copy=False)\n",
        "        xcenter = (t[3]+t[2])/(2*img_dims[key][0])  # (x1+x2)/(2*width)\n",
        "        ycenter = (t[0]+t[1])/(2*img_dims[key][1])  # (y1+y2)/(2*height)\n",
        "        width = (t[3]-t[2])/img_dims[key][0]  # (x2-x1)/(2*width)\n",
        "        height = (t[1]-t[0])/img_dims[key][1]  # (y2-y1)/(2*height)\n",
        "        print(1, xcenter, ycenter, width, height, file=yolo_f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_vGDc-GPX2lF",
        "colab_type": "text"
      },
      "source": [
        "## Install Yolo & prepare to train custom objects\n",
        "https://github.com/AlexeyAB/darknet/#how-to-train-to-detect-your-custom-objects"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-p6OBAGaioms",
        "colab_type": "text"
      },
      "source": [
        "### Install Yolo AlexeyAB fork and prepare metadata"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xVUwORfdY1oY",
        "colab": {}
      },
      "source": [
        "%cd /content/\n",
        "!rm -fr darknet\n",
        "!git clone https://github.com/AlexeyAB/darknet/\n",
        "%cd darknet"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "slVSP1izYHjq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd /content/darknet\n",
        "#!mkdir build-release\n",
        "#%cd build-release\n",
        "#!cmake ..\n",
        "#!make\n",
        "#!make install\n",
        "%cp \"$proj_path\"Makefile ./\n",
        "!make"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2iI-m64EbuQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp \"$data_path\"/yolo_cfg/yolo-obj.cfg /content/darknet/build/darknet/x64/cfg/\n",
        "!cp \"$data_path\"/yolo_cfg/obj.names /content/darknet/build/darknet/x64/data/\n",
        "!cp \"$data_path\"/yolo_cfg/obj.data /content/darknet/build/darknet/x64/data/\n",
        "!cp \"$data_path\"/yolo_cfg/train.txt /content/darknet/build/darknet/x64/data/\n",
        "!cp \"$data_path\"/yolo_cfg/valid.txt /content/darknet/build/darknet/x64/data/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQKUGC1ItZUj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp /content/bib_racer/data/yolov3.weights /content/darknet/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YAaNQp_QTQaD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#%mkdir /content/darknet/build/darknet/x64/data/obj_train/\n",
        "%mkdir /content/darknet/build/darknet/x64/data/obj_valid/\n",
        "#!cp \"$data_path\"/yolo_train/*.* /content/darknet/build/darknet/x64/data/obj_train/ -v"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cKceZL9r_GLj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!cp \"$proj_path\"data/images/train_tiny_50files/*.* /content/darknet/build/darknet/x64/data/obj_valid/ -v"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N-IC4SIcW21o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!wget -O \"$proj_path\"darknet53.conv.74 https://pjreddie.com/media/files/darknet53.conv.74"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGvvhEVyyBIn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!cp \"$data_path\"/yolo_cfg/obj.data /content/darknet/build/darknet/x64/data/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LD_OJ4bkq0D_",
        "colab_type": "text"
      },
      "source": [
        "###Train the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eoJlq7plSWiY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%cd /content/darknet/\n",
        "#!./darknet detector train ./build/darknet/x64/data/obj.data ./build/darknet/x64/cfg/yolo-obj.cfg \"$proj_path\"darknet53.conv.74\n",
        "!./darknet detector train ./build/darknet/x64/data/obj.data ./build/darknet/x64/cfg/yolo-obj.cfg /content/bib_racer/data/RBNR_data/weights_backup_resized_RBNR_training_images/yolo-obj_last.weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZApL-Vkt7vig",
        "colab_type": "text"
      },
      "source": [
        "## Testing YOLO with RBNR dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFoLunbunh9d",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "456c4159-d20a-4cc2-8db4-cbeeb46c7d3f"
      },
      "source": [
        "# Prepare validation images and weight files\n",
        "%cd /content/darknet/\n",
        "![ ! -d /content/darknet/build/darknet/x64/data/obj_valid/ ] && mkdir /content/darknet/build/darknet/x64/data/obj_valid/ && cp \"$data_path\"/yolo_valid/*.* /content/darknet/build/darknet/x64/data/obj_valid/\n",
        "WEIGHT_FILE = \"yolo-obj_4000.weights\"\n",
        "![ ! -f $WEIGHT_FILE ] && cp \"$proj_path\"data/RBNR_data/weights_backup_resized_RBNR_training_images/$WEIGHT_FILE . -v"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/darknet\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I-wZ7EmZjHL1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Test\n",
        "#!./darknet detector test ./build/darknet/x64/data/obj.data ./build/darknet/x64/cfg/yolo-obj.cfg $WEIGHT_FILE -thresh 0.25 -i 0 -ext_output\n",
        "#!./darknet detector map ./build/darknet/x64/data/obj.data ./build/darknet/x64/cfg/yolo-obj.cfg $WEIGHT_FILE\n",
        "!./darknet detector test ./cfg/coco.data ./cfg/yolov3.cfg ./yolov3.weights -thresh 0.25"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vQ8BHokeY4zl",
        "colab_type": "text"
      },
      "source": [
        "## Face and Bib detection with Yolo\n",
        "http://emaraic.com/blog/yolov3-custom-object-detector"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8h0idJtJ_N72",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    }
  ]
}