multi_object_detection_with_Tensorflow_object_detection_API.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "multi object detection with Tensorflow object detection API.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMxdXRwyADqH5MLvVoleWOp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Nnamaka/multi-object-detection-with-tf/blob/main/multi_object_detection_with_Tensorflow_object_detection_API.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Multi Object Detection Using the Tensorflow Object Detection API"
      ],
      "metadata": {
        "id": "XSopwAOdz77t"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##creating our folder structure"
      ],
      "metadata": {
        "id": "tbkrACOr2LCW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os"
      ],
      "metadata": {
        "id": "Q7YfrpMZ0Do1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip list | grep opencv "
      ],
      "metadata": {
        "id": "gvCMxSMbo7NS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#!pip install opencv-contrib-python 4.1.2.30\n",
        "!pip install --upgrade opencv-contrib-python\n",
        "\n",
        "!pip uninstall opencv-python==4.1.2.30\n",
        "!pip install opencv-python==4.5.5.64\n",
        "\n",
        "!pip uninstall opencv-python-headless==4.1.2.30\n",
        "!pip install opencv-python-headless==4.5.5.64"
      ],
      "metadata": {
        "id": "v1lY9MOypDQR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install imgaug==0.2.5\n"
      ],
      "metadata": {
        "id": "bRy8-StiuHps"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip list | grep opencv "
      ],
      "metadata": {
        "id": "OE8KgBLxr9Zv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "CUSTOM_MODEL_NAME = 'my_ssd_mobnet' \n",
        "PRETRAINED_MODEL_NAME = 'ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8'\n",
        "PRETRAINED_MODEL_URL = 'http://download.tensorflow.org/models/object_detection/tf2/20200711/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8.tar.gz'\n",
        "TF_RECORD_SCRIPT_NAME = 'generate_tfrecord.py'\n",
        "LABEL_MAP_NAME = 'label_map.pbtxt'"
      ],
      "metadata": {
        "id": "PJiWTFgK2PwT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paths = {\n",
        "    'WORKSPACE_PATH': os.path.join('Tensorflow', 'workspace'),\n",
        "    'SCRIPTS_PATH': os.path.join('Tensorflow','scripts'),\n",
        "    'APIMODEL_PATH': os.path.join('Tensorflow','models'),\n",
        "    'ANNOTATION_PATH': os.path.join('Tensorflow', 'workspace','annotations'),\n",
        "    'IMAGE_PATH': os.path.join('Tensorflow', 'workspace','images'),\n",
        "    'MODEL_PATH': os.path.join('Tensorflow', 'workspace','models'),\n",
        "    'PRETRAINED_MODEL_PATH': os.path.join('Tensorflow', 'workspace','pre-trained-models'),\n",
        "    'CHECKPOINT_PATH': os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME), \n",
        "    'OUTPUT_PATH': os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME, 'export'), \n",
        "    'TFJS_PATH':os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME, 'tfjsexport'), \n",
        "    'TFLITE_PATH':os.path.join('Tensorflow', 'workspace','models',CUSTOM_MODEL_NAME, 'tfliteexport'), \n",
        "    'PROTOC_PATH':os.path.join('Tensorflow','protoc')\n",
        " }"
      ],
      "metadata": {
        "id": "RxkR_hjP2QBP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "files = {\n",
        "    'PIPELINE_CONFIG':os.path.join('Tensorflow', 'workspace','models', CUSTOM_MODEL_NAME, 'pipeline.config'),\n",
        "    'TF_RECORD_SCRIPT': os.path.join(paths['SCRIPTS_PATH'], TF_RECORD_SCRIPT_NAME), \n",
        "    'LABELMAP': os.path.join(paths['ANNOTATION_PATH'], LABEL_MAP_NAME)\n",
        "}"
      ],
      "metadata": {
        "id": "3a4ZzFdk2ZAG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for path in paths.values():\n",
        "    if not os.path.exists(path):\n",
        "        if os.name == 'posix':\n",
        "            !mkdir -p {path}\n",
        "        if os.name == 'nt':\n",
        "            !mkdir {path}"
      ],
      "metadata": {
        "id": "-8Cl7CdG2ZGp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Download Tensorflow pretrained models and Install TFOD(Tensorflow object detection)"
      ],
      "metadata": {
        "id": "wn6KCqZ_2mKi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if not os.path.exists(os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection')):\n",
        "    !git clone https://github.com/tensorflow/models {paths['APIMODEL_PATH']}"
      ],
      "metadata": {
        "id": "nm_n6tHqgexu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install TFOD"
      ],
      "metadata": {
        "id": "zVGbRo4Hg-j1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if os.name=='posix':  \n",
        "    !apt-get install protobuf-compiler\n",
        "    !cd Tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=. && cp object_detection/packages/tf2/setup.py . && python -m pip install . \n",
        "    \n",
        "if os.name=='nt':\n",
        "    url=\"https://github.com/protocolbuffers/protobuf/releases/download/v3.15.6/protoc-3.15.6-win64.zip\"\n",
        "    wget.download(url)\n",
        "    !move protoc-3.15.6-win64.zip {paths['PROTOC_PATH']}\n",
        "    !cd {paths['PROTOC_PATH']} && tar -xf protoc-3.15.6-win64.zip\n",
        "    os.environ['PATH'] += os.pathsep + os.path.abspath(os.path.join(paths['PROTOC_PATH'], 'bin'))   \n",
        "    !cd Tensorflow/models/research && protoc object_detection/protos/*.proto --python_out=. && copy object_detection\\\\packages\\\\tf2\\\\setup.py setup.py && python setup.py build && python setup.py install\n",
        "    !cd Tensorflow/models/research/slim && pip install -e . "
      ],
      "metadata": {
        "id": "Ngc0vb0DhF-L",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install dill==0.3.4 cloudpickle==1.2.0 requests==2.23.0 folium==0.2.1"
      ],
      "metadata": {
        "id": "DnzYFvDVnduA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run verification script"
      ],
      "metadata": {
        "id": "TEmYdlhEhXyY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "VERIFICATION_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'builders', 'model_builder_tf2_test.py')\n",
        "# Verify Installation\n",
        "!python {VERIFICATION_SCRIPT}"
      ],
      "metadata": {
        "id": "fo-oqlshhZ_v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Install tensorflow if necessary"
      ],
      "metadata": {
        "id": "yboCckH8hj5K"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflow --upgrade"
      ],
      "metadata": {
        "id": "6yspth26hmiO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This cell was for debbugging for errors for matplotlib\n",
        "# !pip uninstall protobuf matplotlib -y\n",
        "# !pip install protobuf matplotlib==3.2"
      ],
      "metadata": {
        "id": "u0g2TYviiX6W"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import the detection api for sanity check"
      ],
      "metadata": {
        "id": "bqAGX_B2i_xH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import object_detection"
      ],
      "metadata": {
        "id": "f4uwE2rDjE1k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download pretrained model"
      ],
      "metadata": {
        "id": "j34iTYlEieXA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if os.name =='posix':\n",
        "    !wget {PRETRAINED_MODEL_URL}\n",
        "    !mv {PRETRAINED_MODEL_NAME+'.tar.gz'} {paths['PRETRAINED_MODEL_PATH']}\n",
        "    !cd {paths['PRETRAINED_MODEL_PATH']} && tar -zxvf {PRETRAINED_MODEL_NAME+'.tar.gz'}\n",
        "if os.name == 'nt':\n",
        "    wget.download(PRETRAINED_MODEL_URL)\n",
        "    !move {PRETRAINED_MODEL_NAME+'.tar.gz'} {paths['PRETRAINED_MODEL_PATH']}\n",
        "    !cd {paths['PRETRAINED_MODEL_PATH']} && tar -zxvf {PRETRAINED_MODEL_NAME+'.tar.gz'}"
      ],
      "metadata": {
        "id": "7PkxcEKuiWTm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Create Label map"
      ],
      "metadata": {
        "id": "PCB3BvmViDQV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "labels = [{'name':'ThumbsUp', 'id':1}, {'name':'ThumbsDown', 'id':2}, {'name':'ThankYou', 'id':3}, {'name':'LiveLong', 'id':4}]\n",
        "\n",
        "with open(files['LABELMAP'], 'w') as f:\n",
        "    for label in labels:\n",
        "        f.write('item { \\n')\n",
        "        f.write('\\tname:\\'{}\\'\\n'.format(label['name']))\n",
        "        f.write('\\tid:{}\\n'.format(label['id']))\n",
        "        f.write('}\\n')"
      ],
      "metadata": {
        "id": "HBJ7RuzXiF16"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Create TF records"
      ],
      "metadata": {
        "id": "UhrW3YyTjc5p"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "collect image from google drive"
      ],
      "metadata": {
        "id": "Gt4L4itRzMug"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "b6qSVBBuzSGS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "copy compressed image data from google drive into the  google colab workspace data folder.\n",
        "\n",
        "Note: I saved my image dataset including my annotation in a folder I named 'TFOD images' in my google drive. This compressed data contains my custom images for a specific object detection task. The dataset already contains an xml file annotations per image and is already shared 80/20"
      ],
      "metadata": {
        "id": "9bMXBrkBdYK0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!cp '/content/drive/MyDrive/TFOD images/imgs.tar.gz' {paths['IMAGE_PATH']}\n"
      ],
      "metadata": {
        "id": "pUkkSmiTdhYE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "uncompress the compressed file"
      ],
      "metadata": {
        "id": "dO7EdVYSkPAQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ARCHIVE_FILES = os.path.join(paths['IMAGE_PATH'], 'imgs.tar.gz')\n",
        "if os.path.exists(ARCHIVE_FILES):\n",
        "  !tar -zxvf {ARCHIVE_FILES}\n",
        "  !mv '/content/test' '/content/train' {paths['IMAGE_PATH']}"
      ],
      "metadata": {
        "id": "duvwqY8Zjbhl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "get TF_RECORD_SCRIPT"
      ],
      "metadata": {
        "id": "MbVvVCtFmP9-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if not os.path.exists(files['TF_RECORD_SCRIPT']):\n",
        "    !git clone https://github.com/nicknochnack/GenerateTFRecord {paths['SCRIPTS_PATH']}"
      ],
      "metadata": {
        "id": "2qEmEguqmITA",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python {files['TF_RECORD_SCRIPT']} -x {os.path.join(paths['IMAGE_PATH'], 'train')} -l {files['LABELMAP']} -o {os.path.join(paths['ANNOTATION_PATH'], 'train.record')} \n",
        "!python {files['TF_RECORD_SCRIPT']} -x {os.path.join(paths['IMAGE_PATH'], 'test')} -l {files['LABELMAP']} -o {os.path.join(paths['ANNOTATION_PATH'], 'test.record')} "
      ],
      "metadata": {
        "id": "qz0UHTwLmOSO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Copy Model config file to training folder"
      ],
      "metadata": {
        "id": "eEukfQL4opaj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "if os.name =='posix':\n",
        "    !cp {os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'pipeline.config')} {os.path.join(paths['CHECKPOINT_PATH'])}\n",
        "if os.name == 'nt':\n",
        "    !copy {os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'pipeline.config')} {os.path.join(paths['CHECKPOINT_PATH'])}"
      ],
      "metadata": {
        "id": "U9TDig6Aot3O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Update config for transfer Learning"
      ],
      "metadata": {
        "id": "Jx-C0rk3o2Y2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from object_detection.utils import config_util\n",
        "from object_detection.protos import pipeline_pb2\n",
        "from google.protobuf import text_format"
      ],
      "metadata": {
        "id": "jrb4zrvko7G8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "config = config_util.get_configs_from_pipeline_file(files['PIPELINE_CONFIG'])"
      ],
      "metadata": {
        "id": "Wv-oruhQpDlf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "config"
      ],
      "metadata": {
        "id": "sRLicWiCpvFM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()\n",
        "with tf.io.gfile.GFile(files['PIPELINE_CONFIG'], \"r\") as f:                                                                                                                                                                                                                     \n",
        "    proto_str = f.read()                                                                                                                                                                                                                                          \n",
        "    text_format.Merge(proto_str, pipeline_config)  "
      ],
      "metadata": {
        "id": "Qew7BvlupQb3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline_config.model.ssd.num_classes = len(labels)\n",
        "pipeline_config.train_config.batch_size = 4\n",
        "pipeline_config.train_config.fine_tune_checkpoint = os.path.join(paths['PRETRAINED_MODEL_PATH'], PRETRAINED_MODEL_NAME, 'checkpoint', 'ckpt-0')\n",
        "pipeline_config.train_config.fine_tune_checkpoint_type = \"detection\"\n",
        "pipeline_config.train_input_reader.label_map_path= files['LABELMAP']\n",
        "pipeline_config.train_input_reader.tf_record_input_reader.input_path[:] = [os.path.join(paths['ANNOTATION_PATH'], 'train.record')]\n",
        "pipeline_config.eval_input_reader[0].label_map_path = files['LABELMAP']\n",
        "pipeline_config.eval_input_reader[0].tf_record_input_reader.input_path[:] = [os.path.join(paths['ANNOTATION_PATH'], 'test.record')]"
      ],
      "metadata": {
        "id": "OC3exaXLpsWk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "config_text = text_format.MessageToString(pipeline_config)                                                                                                                                                                                                        \n",
        "with tf.io.gfile.GFile(files['PIPELINE_CONFIG'], \"wb\") as f:                                                                                                                                                                                                                     \n",
        "    f.write(config_text)  "
      ],
      "metadata": {
        "id": "zN063tmHpTkP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Train the model"
      ],
      "metadata": {
        "id": "KDtnKcZ4pIoC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "TRAINING_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'model_main_tf2.py')"
      ],
      "metadata": {
        "id": "_BlE4H1OpEfE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "command = \"python {} --model_dir={} --pipeline_config_path={} --num_train_steps=2000\".format(TRAINING_SCRIPT, paths['CHECKPOINT_PATH'],files['PIPELINE_CONFIG'])"
      ],
      "metadata": {
        "id": "KXUuSTc9p-zS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "tJHYH4QzvChf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(command)"
      ],
      "metadata": {
        "id": "cFmFkTLIqDPj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "YXFL389dqGkz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Evaluate The Model"
      ],
      "metadata": {
        "id": "aU0yhZpQP_xV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "command = \"python {} --model_dir={} --pipeline_config_path={} --checkpoint_dir={}\".format(TRAINING_SCRIPT, paths['CHECKPOINT_PATH'],files['PIPELINE_CONFIG'], paths['CHECKPOINT_PATH'])"
      ],
      "metadata": {
        "id": "45OvBPgQPlOi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(command)"
      ],
      "metadata": {
        "id": "xK76tlMjRoST"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "JLPTwzEfRsiw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Load Trained model from checkpoint"
      ],
      "metadata": {
        "id": "h52VJE37Rz9P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import tensorflow as tf\n",
        "from object_detection.utils import label_map_util\n",
        "from object_detection.utils import visualization_utils as viz_utils\n",
        "from object_detection.builders import model_builder\n",
        "from object_detection.utils import config_util"
      ],
      "metadata": {
        "id": "J6JKgOX_Ru_k"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "NOTE: for loading the correct and the latest checkpoint, go into <code>Tensorflow/workspace/models/my_ssd_mobnet</code> and see the number of the last checkpoint, then make the changes accordingly in second argument of <code>ckpt.restore()</code> function"
      ],
      "metadata": {
        "id": "_2mD0PHngjt4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load pipeline config and build a detection model\n",
        "configs = config_util.get_configs_from_pipeline_file(files['PIPELINE_CONFIG'])\n",
        "detection_model = model_builder.build(model_config=configs['model'], is_training=False)\n",
        "\n",
        "# Restore checkpoint\n",
        "ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)\n",
        "ckpt.restore(os.path.join(paths['CHECKPOINT_PATH'], 'ckpt-3')).expect_partial()\n",
        "\n",
        "@tf.function\n",
        "def detect_fn(image):\n",
        "    image, shapes = detection_model.preprocess(image)\n",
        "    prediction_dict = detection_model.predict(image, shapes)\n",
        "    detections = detection_model.postprocess(prediction_dict, shapes)\n",
        "    return detections"
      ],
      "metadata": {
        "id": "kdzlBQ-BR90j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Detect From an image"
      ],
      "metadata": {
        "id": "HGtzhlp1SJJS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import cv2 \n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline"
      ],
      "metadata": {
        "id": "7bpjOmXAS-YI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "category_index = label_map_util.create_category_index_from_labelmap(files['LABELMAP'])"
      ],
      "metadata": {
        "id": "f9zGRObBTF6q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "look inside <code>IMAGE_PATH</code> to select the image you'ld like to infer and copy the name to the <code>name</code> variable below"
      ],
      "metadata": {
        "id": "O57TTnLDTi_2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "name = 'thumbsup.d99669e0-8f12-11ec-9b65-1008b1099b24.jpg'\n",
        "IMAGE_PATH = os.path.join(paths['IMAGE_PATH'], 'test', name)"
      ],
      "metadata": {
        "id": "MS50G6VzTKMU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "img = cv2.imread(IMAGE_PATH)\n",
        "image_np = np.array(img)\n",
        "\n",
        "input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)\n",
        "detections = detect_fn(input_tensor)\n",
        "\n",
        "num_detections = int(detections.pop('num_detections'))\n",
        "detections = {key: value[0, :num_detections].numpy()\n",
        "              for key, value in detections.items()}\n",
        "detections['num_detections'] = num_detections\n",
        "\n",
        "# detection_classes should be ints.\n",
        "detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n",
        "\n",
        "label_id_offset = 1\n",
        "image_np_with_detections = image_np.copy()\n",
        "\n",
        "viz_utils.visualize_boxes_and_labels_on_image_array(\n",
        "            image_np_with_detections,\n",
        "            detections['detection_boxes'],\n",
        "            detections['detection_classes']+label_id_offset,\n",
        "            detections['detection_scores'],\n",
        "            category_index,\n",
        "            use_normalized_coordinates=True,\n",
        "            max_boxes_to_draw=5,\n",
        "            min_score_thresh=.8,\n",
        "            agnostic_mode=False)\n",
        "\n",
        "plt.imshow(cv2.cvtColor(image_np_with_detections, cv2.COLOR_BGR2RGB))\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "8mgEgJLMTWpF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "qb1T3V3vUTYh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Real Time detection\n",
        "Note: This SECTION is optional and should be performed locally after freezing and downloading your model"
      ],
      "metadata": {
        "id": "PH7nAtirUhU_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# opencv-python-headless is already installed else uncomment and run the code below\n",
        "# !pip uninstall opencv-python-headless -y"
      ],
      "metadata": {
        "id": "G5eUpS0ffro6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "uncomment the cell below to begin real time object detection with pc camera"
      ],
      "metadata": {
        "id": "1wKb7oMHjKsV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# cap = cv2.VideoCapture(0)\n",
        "# width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
        "# height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
        "\n",
        "# while cap.isOpened(): \n",
        "#     ret, frame = cap.read()\n",
        "#     image_np = np.array(frame)\n",
        "    \n",
        "#     input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)\n",
        "#     detections = detect_fn(input_tensor)\n",
        "    \n",
        "#     num_detections = int(detections.pop('num_detections'))\n",
        "#     detections = {key: value[0, :num_detections].numpy()\n",
        "#                   for key, value in detections.items()}\n",
        "#     detections['num_detections'] = num_detections\n",
        "\n",
        "#     # detection_classes should be ints.\n",
        "#     detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n",
        "\n",
        "#     label_id_offset = 1\n",
        "#     image_np_with_detections = image_np.copy()\n",
        "\n",
        "#     viz_utils.visualize_boxes_and_labels_on_image_array(\n",
        "#                 image_np_with_detections,\n",
        "#                 detections['detection_boxes'],\n",
        "#                 detections['detection_classes']+label_id_offset,\n",
        "#                 detections['detection_scores'],\n",
        "#                 category_index,\n",
        "#                 use_normalized_coordinates=True,\n",
        "#                 max_boxes_to_draw=5,\n",
        "#                 min_score_thresh=.8,\n",
        "#                 agnostic_mode=False)\n",
        "\n",
        "#     cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))\n",
        "    \n",
        "#     if cv2.waitKey(10) & 0xFF == ord('q'):\n",
        "#         cap.release()\n",
        "#         cv2.destroyAllWindows()\n",
        "#         break"
      ],
      "metadata": {
        "id": "4wZWODjVhxg6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Freezing the Graph"
      ],
      "metadata": {
        "id": "YtFyCQ_claAE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "FREEZE_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'exporter_main_v2.py ')"
      ],
      "metadata": {
        "id": "kcgoj4JVlnzS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "command = \"python {} --input_type=image_tensor --pipeline_config_path={} --trained_checkpoint_dir={} --output_directory={}\".format(FREEZE_SCRIPT ,files['PIPELINE_CONFIG'], paths['CHECKPOINT_PATH'], paths['OUTPUT_PATH'])"
      ],
      "metadata": {
        "id": "t8_7ES1-l0oO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(command)"
      ],
      "metadata": {
        "id": "_EXvhuk9l626"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}f"
      ],
      "metadata": {
        "id": "FlQuaNU-l-oB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Converting model to TFJS"
      ],
      "metadata": {
        "id": "tHuzJBa1qiEZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install tensorflowjs"
      ],
      "metadata": {
        "id": "bfqpFHDuqnzC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "command = \"tensorflowjs_converter --input_format=tf_saved_model --output_node_names='detection_boxes,detection_classes,detection_features,detection_multiclass_scores,detection_scores,num_detections,raw_detection_boxes,raw_detection_scores' --output_format=tfjs_graph_model --signature_name=serving_default {} {}\".format(os.path.join(paths['OUTPUT_PATH'], 'saved_model'), paths['TFJS_PATH'])"
      ],
      "metadata": {
        "id": "vmYOq8_YqtEh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(command)"
      ],
      "metadata": {
        "id": "NONnyWqCqyiV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "JunxtT4Nq1pS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Converting model to TFLite"
      ],
      "metadata": {
        "id": "NIJYRSuNq5lm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "TFLITE_SCRIPT = os.path.join(paths['APIMODEL_PATH'], 'research', 'object_detection', 'export_tflite_graph_tf2.py ')"
      ],
      "metadata": {
        "id": "MLzwIMtzrBSK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "command = \"python {} --pipeline_config_path={} --trained_checkpoint_dir={} --output_directory={}\".format(TFLITE_SCRIPT ,files['PIPELINE_CONFIG'], paths['CHECKPOINT_PATH'], paths['TFLITE_PATH'])"
      ],
      "metadata": {
        "id": "TagXhMaXrGUj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(command)"
      ],
      "metadata": {
        "id": "eBGn4JLErMzS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "XEjoliV-rOCV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "FROZEN_TFLITE_PATH = os.path.join(paths['TFLITE_PATH'], 'saved_model')\n",
        "TFLITE_MODEL = os.path.join(paths['TFLITE_PATH'], 'saved_model', 'detect.tflite')"
      ],
      "metadata": {
        "id": "SuhcQ0DFrUFQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "command = \"tflite_convert \\\n",
        "--saved_model_dir={} \\\n",
        "--output_file={} \\\n",
        "--input_shapes=1,300,300,3 \\\n",
        "--input_arrays=normalized_input_image_tensor \\\n",
        "--output_arrays='TFLite_Detection_PostProcess','TFLite_Detection_PostProcess:1','TFLite_Detection_PostProcess:2','TFLite_Detection_PostProcess:3' \\\n",
        "--inference_type=FLOAT \\\n",
        "--allow_custom_ops\".format(FROZEN_TFLITE_PATH, TFLITE_MODEL, )"
      ],
      "metadata": {
        "id": "hCIK3FKXrbjh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(command)"
      ],
      "metadata": {
        "id": "yeZiy4OIrfrb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!{command}"
      ],
      "metadata": {
        "id": "p6GjFkTvrgtc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Zip and Export Models\n",
        "NOTE: check the directory where <code>models.tar.gz</code> is saved. \n",
        "change into that directory and run the code below."
      ],
      "metadata": {
        "id": "8B8t8ABMrpvC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!tar -czf models.tar.gz {paths['CHECKPOINT_PATH']}"
      ],
      "metadata": {
        "id": "s9Dm5LUOrvOK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "3dx35ZcqsJkQ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}