notebooks/evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from io import StringIO\n",
    "from matplotlib import pyplot as plt\n",
    "from PIL import Image\n",
    "import glob\n",
    "import random\n",
    "import time\n",
    "from tqdm import tqdm_notebook\n",
    "from lxml import etree\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_model(file_path):\n",
    "    detection_graph = tf.Graph()\n",
    "    with detection_graph.as_default():\n",
    "        od_graph_def = tf.GraphDef()\n",
    "        with tf.gfile.GFile(file_path, 'rb') as fid:\n",
    "            serialized_graph = fid.read()\n",
    "            od_graph_def.ParseFromString(serialized_graph)\n",
    "            tf.import_graph_def(od_graph_def, name='')\n",
    "    return detection_graph\n",
    "            \n",
    "def load_image(image_path):\n",
    "    image = Image.open(image_path)\n",
    "    (im_width, im_height) = image.size\n",
    "    return np.array(image.getdata()).reshape((im_height, im_width, 3)).astype(np.uint8)\n",
    "\n",
    "def run_inference(sess, ops, image_tensor, image):\n",
    "    output_dict = {}\n",
    "    \n",
    "    time_s = time.time()\n",
    "    num_detections, boxes, scores, classes = sess.run(ops, feed_dict={image_tensor: image})\n",
    "    time_t = time.time() - time_s\n",
    "    \n",
    "    output_dict['num_detections'] = int(num_detections[0])\n",
    "    output_dict['detection_classes'] = classes[0].astype(np.uint8)\n",
    "    output_dict['detection_boxes'] = boxes[0]\n",
    "    output_dict['detection_scores'] = scores[0]\n",
    "    output_dict['detection_time'] = time_t\n",
    "    \n",
    "    return output_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_class(detection_dict, threshold = 0.5):\n",
    "    detection_classes = detection_dict['detection_classes']\n",
    "    detection_scores = detection_dict['detection_scores']\n",
    "    if detection_scores[0] >= threshold:\n",
    "        return detection_classes[0]\n",
    "    return 0\n",
    "\n",
    "def load_labels(image_paths, labels_dir, label_map):\n",
    "    labels = []\n",
    "    for image_path in image_paths:\n",
    "        file_name = os.path.splitext(os.path.basename(image_path))[0]\n",
    "        image_label = os.path.join(labels_dir, file_name + '.xml')\n",
    "        image_class = 0 # Fake class for \"background\" images without traffic lights\n",
    "\n",
    "        if os.path.isfile(image_label):\n",
    "            with tf.gfile.GFile(image_label, 'r') as f:\n",
    "                xml = f.read()\n",
    "            xml = etree.fromstring(xml)\n",
    "            object_el = next(xml.iter('object'), None)\n",
    "            if object_el is not None:\n",
    "                class_name = next(object_el.iter('name')).text\n",
    "                image_class = label_map[class_name]\n",
    "\n",
    "        labels.append(image_class)\n",
    "    return labels\n",
    "\n",
    "def evaluate(name, graph, image_paths, labels, cpu = False):\n",
    "    \n",
    "    predictions = []\n",
    "    \n",
    "    with graph.as_default():\n",
    "    \n",
    "        image_tensor = graph.get_tensor_by_name('image_tensor:0')\n",
    "        boxes_tensor = graph.get_tensor_by_name('detection_boxes:0')\n",
    "        scores_tensor = graph.get_tensor_by_name('detection_scores:0')\n",
    "        classes_tensor = graph.get_tensor_by_name('detection_classes:0')\n",
    "        detections_tensor = graph.get_tensor_by_name('num_detections:0')\n",
    "\n",
    "        ops = [detections_tensor, boxes_tensor, scores_tensor, classes_tensor]\n",
    "        \n",
    "        config = None\n",
    "        \n",
    "        if cpu:\n",
    "            config = tf.ConfigProto(device_count = {'GPU': 0})\n",
    "\n",
    "        with tf.Session(config=config) as sess:\n",
    "            \n",
    "            if len(image_paths) > 0:\n",
    "                # Preload\n",
    "                image = load_image(image_paths[0])\n",
    "                run_inference(sess, ops, image_tensor, np.expand_dims(image, axis=0))\n",
    "            \n",
    "            time_t = 0\n",
    "            for image_path in tqdm_notebook(image_paths, desc='{}'.format(name), unit=' images'):\n",
    "                image = load_image(image_path)\n",
    "                # Expand dimensions since the model expects images to have shape: [1, None, None, 3]\n",
    "                image = np.expand_dims(image, axis=0)\n",
    "                # Actual detection.\n",
    "                output_dict = run_inference(sess, ops, image_tensor, image)\n",
    "                time_t += output_dict['detection_time']\n",
    "                prediction = extract_class(output_dict)\n",
    "                predictions.append(prediction)\n",
    "                \n",
    "    for i, (label, prediction) in enumerate(zip(labels, predictions)):\n",
    "        if label != prediction:\n",
    "            print('Image misclassified: {} (Expected: {}, Predicted: {})'.format(image_paths[i], label, prediction))\n",
    "                \n",
    "    accuracy = accuracy_score(labels, predictions)\n",
    "    \n",
    "    return accuracy, time_t * 1000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABELS_MAP_PATH = os.path.join('..', 'config', 'labels_map.pbtxt')\n",
    "LABELS_MAP = {'Green': 1, 'Yellow': 2, 'Red': 3}\n",
    "\n",
    "def evaluate_models(models_dir, data_dir, models, model_file = 'frozen_inference_graph.pb', n_samples = None, cpu = False):\n",
    "    image_paths = glob.glob(os.path.join(data_dir, '*.jpg'))\n",
    "    labels = load_labels(image_paths, os.path.join(data_dir, 'labels'), LABELS_MAP)\n",
    "    image_paths, labels = shuffle(image_paths, labels, n_samples = n_samples)\n",
    "    \n",
    "    for model_name in models:\n",
    "        model_path = os.path.join(models_dir, model_name, model_file)\n",
    "        graph = load_model(model_path)\n",
    "        accuracy, total_time = evaluate(model_name, graph, image_paths, labels, cpu = cpu)\n",
    "        avg_time = total_time / len(image_paths)\n",
    "        fps = 1000 / avg_time\n",
    "        print('Model {} Accuracy: {:.3f} (Avg Time: {:.3f} ms, FPS: {:.3f})'.format(model_name, accuracy, avg_time, fps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODELS_DIR = os.path.join('..', 'models', 'exported')\n",
    "DATA_DIR = os.path.join('..', 'data', 'simulator')\n",
    "\n",
    "n_samples = 500\n",
    "\n",
    "models = ['ssd_mobilenet_v1', 'ssd_mobilenet_v2', 'ssdlite_mobilenet_v2', 'ssd_inception_v2']\n",
    "\n",
    "evaluate_models(MODELS_DIR, DATA_DIR, models, n_samples = n_samples, cpu = False)\n",
    "\n",
    "# Performance eval\n",
    "\n",
    "n_samples = 50\n",
    "\n",
    "evaluate_models(MODELS_DIR, DATA_DIR, models, n_samples = n_samples, cpu = False)\n",
    "evaluate_models(MODELS_DIR, DATA_DIR, models, n_samples = n_samples, cpu = True)\n",
    "\n",
    "# Performance eval on optimized models\n",
    "\n",
    "MODELS_DIR = os.path.join('..', 'models', 'exported_optimized')\n",
    "\n",
    "evaluate_models(MODELS_DIR, DATA_DIR, models, model_file = 'graph_optimized.pb', n_samples = n_samples, cpu = False)\n",
    "evaluate_models(MODELS_DIR, DATA_DIR, models, model_file = 'graph_optimized.pb', n_samples = n_samples, cpu = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}