MVSEC_extract_images_day2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "name": "MVSEC_extract_images_day2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "2TKp2LTMDC9s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8c8fdd2f-9206-40cf-fbb9-d6d481cfc514"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rqMuS39qW2Up"
      },
      "source": [
        "from PIL import Image as im\n",
        "import os\n",
        "import cv2\n",
        "import math\n",
        "import scipy.io\n",
        "import random\n",
        "import networkx\n",
        "import h5py\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oR_LBwebDHdg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e2514f94-bb43-47ac-96cd-cd5a2ae05903"
      },
      "source": [
        "file_name = \"/content/drive/MyDrive/hdf5/outdoor_day/outdoor_day2_data.hdf5\"\n",
        "\n",
        "f = h5py.File(file_name, \"r\")\n",
        "for key in f.keys():\n",
        "  print(key)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "davis\n",
            "velodyne\n",
            "visensor\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T3GlUTgNDv47",
        "outputId": "bef64496-95ab-4022-946e-9046eac3935a"
      },
      "source": [
        "davis = f['davis']\n",
        "for key in davis.keys():\n",
        "  print(key)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "left\n",
            "right\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DDfJmEfmD3xf",
        "outputId": "1e3eed5d-d0c8-407d-971b-b2bbd67818dd"
      },
      "source": [
        "left = davis['left']\n",
        "for key in left.keys():\n",
        "  print(key,end=' = ')\n",
        "  print(len(left[str(key)]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "events = 466410069\n",
            "image_raw = 28583\n",
            "image_raw_event_inds = 28583\n",
            "image_raw_ts = 28583\n",
            "imu = 653760\n",
            "imu_ts = 653760\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8uSNYVIWC_W"
      },
      "source": [
        "events = left['events']\n",
        "image_raw = left['image_raw']\n",
        "image_raw_ts = left['image_raw_ts']\n",
        "image_raw_event_inds = left['image_raw_event_inds']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OgO5Lj_0D9sU"
      },
      "source": [
        "image_raw_sample = image_raw[1:101]\n",
        "image_raw_event_inds_sample = image_raw_event_inds[1:101]\n",
        "image_raw_ts_sample = image_raw_ts[1:101]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cocvDgIvVKFU"
      },
      "source": [
        "image_raw_batch = np.asarray(image_raw_sample)\n",
        "image_raw_ts_batch = np.asarray(image_raw_ts_sample)\n",
        "image_raw_event_inds_batch = np.asarray(image_raw_event_inds_sample)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n4NVRP49Bllq"
      },
      "source": [
        "Each image is of the dimension 260 x 346\n",
        "\n",
        "Each pixel has a gray value between [0,256)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xm5p5f6aWrox",
        "outputId": "9ea571b6-da55-47b6-df10-30dd51c8bf2f"
      },
      "source": [
        "img = im.fromarray(image_raw_batch[0]);\n",
        "image_raw_batch[0].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(260, 346)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ckixf1eY3D7I"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZZDGa9EMXEVg",
        "outputId": "9dca990a-bfd8-42c4-ae48-4e4979303919"
      },
      "source": [
        "image_raw_ts_batch[0],image_raw_ts_batch[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1506119776.3952818, 1506119776.418141)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w3dwOvvsYCiT",
        "outputId": "14f2a29c-7790-4f9b-8e9d-3ab66947202e"
      },
      "source": [
        "image_raw_event_inds_batch[0], image_raw_event_inds_batch[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(37, 124)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fnto9JCHD855",
        "outputId": "53b7b854-95bc-4e56-f667-d7a34f90e7a8"
      },
      "source": [
        "A = image_raw_batch[0]\n",
        "B = image_raw_batch[1]\n",
        "(A==B).all()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tPhc_dUlXN8l"
      },
      "source": [
        "# for saving the batch of images. Do not run otherwise\n",
        "#for i in range(100):\n",
        "#  x = str(i//10)\n",
        "#  y = str(i%10)\n",
        "#  np.save(os.path.join('image_raw_batch_as_npy','image_raw_00'+x+y),image_raw_batch)\n",
        "#  np.save(os.path.join('image_raw_ts_batch_as_npy','image_raw_ts_00'+x+y),image_raw_ts_batch)\n",
        "#  np.save(os.path.join('image_raw_event_inds_batch_as_npy','image_raw_event_inds_00'+x+y),image_raw_event_inds_batch)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dAWRpZ0JZT9A"
      },
      "source": [
        "# for creating a zip file. Do not run otherwise\n",
        "# !zip -r image_raw_batch_as_npy.zip /content/image_raw_batch_as_npy\n",
        "# !zip -r image_raw_event_inds_batch_as_npy.zip /content/image_raw_event_inds_batch_as_npy\n",
        "# !zip -r image_raw_ts_batch_as_npy /content/image_raw_ts_batch_as_npy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mj000edcaxfa"
      },
      "source": [
        "# for moving a file. Do not otherwise\n",
        "# !mv /content/image_raw_batch_as_npy.zip /content/drive/MyDrive/Labelling_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVH3MYzgcUc_"
      },
      "source": [
        "# !mv /content/image_raw_event_inds_batch_as_npy.zip /content/drive/MyDrive/Labelling_data\n",
        "# !mv /content/image_raw_ts_batch_as_npy.zip /content/drive/MyDrive/Labelling_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKXIwT29DWmN"
      },
      "source": [
        "\n",
        "# stacked_img = np.stack((s,)*3, axis=-1)\n",
        "# print(stacked_img.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3fcgaPgXc1Bx"
      },
      "source": [
        "\"\"\"for i in range(len(image_raw_batch)):\n",
        "  gray = image_raw_batch[i]\n",
        "  # convert 1 channel grayscale to 3 channel RGB grayscale\n",
        "  gray_rgb = np.stack((gray,)*3, axis=-1)\n",
        "  # convert to IMAGE\n",
        "  img = im.fromarray(gray_rgb)\n",
        "  # histogram equalization\n",
        "  img_to_yuv = cv2.cvtColor(gray_rgb,cv2.COLOR_BGR2YUV)\n",
        "  img_to_yuv[:,:,0] = cv2.equalizeHist(img_to_yuv[:,:,0])\n",
        "  hist_equalization_result = cv2.cvtColor(img_to_yuv, cv2.COLOR_YUV2BGR)\n",
        "  x = str(i//10)\n",
        "  y = str(i%10)\n",
        "  np.save(os.path.join('/content/image_raw_batch_histeq_as_npy','image_raw_batch_histeq_0'+x+y),hist_equalization_result)\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xrScz9osvkwR"
      },
      "source": [
        "# !zip -r image_raw_batch_histeq_as_npy.zip '/content/image_raw_batch_histeq_as_npy'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XcMEI1imEY5U"
      },
      "source": [
        "# !mv /content/image_raw_batch_histeq_as_npy.zip /content/drive/MyDrive/Labelling_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hEp9ZqiSHHKY"
      },
      "source": [
        "\"\"\"for i in range(100):\n",
        "  x = str(i//10)\n",
        "  y = str(i%10)\n",
        "  array = np.load('/content/image_raw_batch_histeq_as_npy/image_raw_batch_histeq_0'+x+y+'.npy')\n",
        "  img = im.fromarray(array)\n",
        "  img.save('/content/images_raw_batch_histeq/image_raw_batch_histeq_0'+x+y+'.png','PNG')\"\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8zEvygCHiIL"
      },
      "source": [
        "#!zip -r images_raw_batch_histeq.zip '/content/images_raw_batch_histeq'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1EHS0iSVKPBR"
      },
      "source": [
        "#!mv /content/images_raw_batch_histeq.zip /content/drive/MyDrive/Labelling_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Z7MhvlGKWJq"
      },
      "source": [
        "A = np.load('/content/image_raw_batch_histeq_as_npy/image_raw_batch_histeq_000.npy')\n",
        "B = np.load('/content/image_raw_batch_histeq_as_npy/image_raw_batch_histeq_001.npy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "euX2K_fLF7BB",
        "outputId": "60156fff-67e1-446a-ffe2-e84050d95eb1"
      },
      "source": [
        "np.array_equal(A,A), np.array_equal(A,B)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(True, False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MfqGTgqAF9Cc",
        "outputId": "e02c090d-ddba-4ed7-cc5b-18622e8d932f"
      },
      "source": [
        "print(image_raw_ts_batch[99],image_raw_ts_batch[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1506119778.6583588 1506119776.3952818\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2JxTwp3itvOU"
      },
      "source": [
        "for i in image_raw_event_inds_batch[:100]:\n",
        "  print(i)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ax4BQp-fHnYP"
      },
      "source": [
        "last = 0\n",
        "for ind in image_raw_event_inds_batch[:1]:\n",
        "  x = '0000000000'\n",
        "  y = str(ind)\n",
        "  file_name = x[:len(x)-len(y)] + y\n",
        "  np.save(os.path.join('/content','event_'+file_name),np.asarray(events[last:ind+1]))\n",
        "  last = ind + 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K6h8f91Jjbk6"
      },
      "source": [
        "#!unzip /content/drive/MyDrive/Labelling_data/events_batch_as_npy.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HCZMHUuvYhA"
      },
      "source": [
        "# !mv /content/events_batch_as_npy.zip /content/drive/MyDrive/Labelling_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ih_H_5xOvf_I"
      },
      "source": [
        "A = np.load('/content/drive/MyDrive/OutdoorDay2asNPY/content/OutdoorDay2asNPY/left_image00037_hist_eq_seg.npy')\n",
        "A[:10]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6H5J1P9LwN5d"
      },
      "source": [
        "#!unzip /content/drive/MyDrive/Labelling_data/image_raw_batch_seg_as_npy.zip -d '/content/Untitled Folder'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KfUrjItfyYul"
      },
      "source": [
        "#!mv '/content/Untitled Folder/content/image_raw_batch_seg_as_npy' /content"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YMHyw6k5DeVe"
      },
      "source": [
        "**Each event is of the form (y_i, x_i, t_i, p_i)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "avhCbaXcymYu"
      },
      "source": [
        "I = np.load('/content/image_raw_batch_seg_as_npy/image_raw_batch_seg_as_npy_000.npy')\n",
        "print(I.shape)  # rows, cols \n",
        "imgI = im.fromarray(I)\n",
        "print(imgI.size)  # width, height"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W6doGLpEFyAA"
      },
      "source": [
        "#color_to_object[tuple(I[114][40])]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s32Mk8GOzHZ-"
      },
      "source": [
        "object_to_color = {\n",
        "    \"background\"  : (0, 0, 0),\n",
        "    \"person\"      : (192, 128, 128),\n",
        "    \"car\"         : (128, 128, 128)\n",
        "}\n",
        "color_to_object = { v : k for k, v in object_to_color.items()}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tIAHZ2NGzPW3"
      },
      "source": [
        "events = np.load('/content/content/events_batch_as_npy/event_0000000037.npy');\n",
        "no_events = len(events)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e2aRTkF-BuJF"
      },
      "source": [
        "for event in events:\n",
        " y, x, t, p = event \n",
        " print(color_to_object[tuple(I[int(x)][int(y)])])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "81LnfkLN34OU"
      },
      "source": [
        "#Classification\n",
        "Background -or- Car -or- Person\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FXlboX_XFc4F"
      },
      "source": [
        "def label_data(events):\n",
        "  object_to_color = {\n",
        "    \"background\"  : (0, 0, 0),\n",
        "    \"person\"      : (192, 128, 128),\n",
        "    \"car\"         : (128, 128, 128)\n",
        "  }\n",
        "  color_to_object = { v : k for k, v in object_to_color.items()}\n",
        "  GT = []\n",
        "  for event in events:\n",
        "    y, x, t, p = event\n",
        "    if color_to_object[tuple(I[int(x)][int(y)])] == \"person\":\n",
        "      GT.append(list((0, 0, 1)))\n",
        "      continue\n",
        "    elif color_to_object[tuple(I[int(x)][int(y)])] == \"car\":\n",
        "      GT.append(list((0, 1, 0)))\n",
        "      continue\n",
        "    GT.append(list((1, 0, 0)))\n",
        "  return np.array(GT)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i8kMqdws4iXe"
      },
      "source": [
        "print(class_tensor(events))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I609mKq747jg"
      },
      "source": [
        "def isWithin(event1, event2, radius, alpha=1, beta=0.000005):\n",
        "    y_i, x_i, t_i, p_i = event1\n",
        "    y_j, x_j, t_j, p_j = event2\n",
        "    t_i = t_i/10000\n",
        "    t_j = t_j/10000\n",
        "    spatial = (x_i - x_j)*(x_i - x_j) + (y_i - y_j)*(y_i - y_j)\n",
        "    temporal = (t_i - t_j)*(t_i - t_j)\n",
        "    if (alpha * spatial) + (beta * temporal * 100000000) >= 0:\n",
        "      #print(math.sqrt(alpha * spatial + beta * temporal))\n",
        "      return math.sqrt(alpha * spatial + beta * temporal) <= radius"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rf6-TuNwAanZ"
      },
      "source": [
        "def visualize_graph(adMat):\n",
        "  M = len(adMat)\n",
        "  labeldict = {}\n",
        "  for i in range(M):\n",
        "    labeldict[i] = str(i) + \"+\" if events[i][3] == 1 else str(i) +\"-\"\n",
        "    \n",
        "  G = networkx.from_numpy_matrix(np.array(adMat))\n",
        "\n",
        "  color_map = []\n",
        "  for node in G:\n",
        "    if events[node][3] == -1:\n",
        "      color_map.append(\"red\")\n",
        "    else:\n",
        "      color_map.append(\"green\")\n",
        "  networkx.draw(G,node_color=color_map, labels = labeldict, with_labels=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvaA8jAYA74o"
      },
      "source": [
        "def construct_graph(events, R = 100):\n",
        "  M = len(events)\n",
        "  adMat = np.zeros([M, M], dtype=int)\n",
        "  for row in range(M):\n",
        "      for col in range(M):\n",
        "        if row == col:\n",
        "          continue \n",
        "        if isWithin(events[row], events[col], R):\n",
        "          adMat[row][col] = 1\n",
        "          adMat[col][row] = 1\n",
        "  return adMat"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pE-X7cy8ELjN"
      },
      "source": [
        "def polar_events(events):\n",
        "  return events[:,-1]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z6AgT_cKB5dR"
      },
      "source": [
        "construct_graph(events)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TAb_mW-AC0Vq"
      },
      "source": [
        "visualize_graph(construct_graph(events[2:25]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zPgPyWnGDKzc"
      },
      "source": [
        "label_data(events)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYczJuw5EE14"
      },
      "source": [
        "!unzip /content/drive/MyDrive/Labelling_data/image_raw_batch_histeq_as_npy.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4JJ_okGZVKRw"
      },
      "source": [
        "# Model training for Ground Truth\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J7MchmrLVVq8",
        "outputId": "dfb438ce-365b-45a9-e40e-c38b741337a0"
      },
      "source": [
        "from io import BytesIO\n",
        "import tarfile\n",
        "import tempfile\n",
        "from google.colab import files\n",
        "from six.moves import urllib\n",
        "\n",
        "from matplotlib import gridspec\n",
        "\n",
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RQ11u3ZCVv74"
      },
      "source": [
        "class DeepLabModel(object):\n",
        "  \"\"\"Class to load deeplab model and run inference.\"\"\"\n",
        "\n",
        "  INPUT_TENSOR_NAME = 'ImageTensor:0'\n",
        "  OUTPUT_TENSOR_NAME = 'SemanticPredictions:0'\n",
        "  INPUT_SIZE = 513\n",
        "  FROZEN_GRAPH_NAME = 'frozen_inference_graph'\n",
        "\n",
        "  def __init__(self, tarball_path):\n",
        "    \"\"\"Creates and loads pretrained deeplab model.\"\"\"\n",
        "    self.graph = tf.Graph()\n",
        "\n",
        "    graph_def = None\n",
        "    # Extract frozen graph from tar archive.\n",
        "    tar_file = tarfile.open(tarball_path)\n",
        "    for tar_info in tar_file.getmembers():\n",
        "      if self.FROZEN_GRAPH_NAME in os.path.basename(tar_info.name):\n",
        "        file_handle = tar_file.extractfile(tar_info)\n",
        "        graph_def = tf.GraphDef.FromString(file_handle.read())\n",
        "        break\n",
        "\n",
        "    tar_file.close()\n",
        "\n",
        "    if graph_def is None:\n",
        "      raise RuntimeError('Cannot find inference graph in tar archive.')\n",
        "\n",
        "    with self.graph.as_default():\n",
        "      tf.import_graph_def(graph_def, name='')\n",
        "\n",
        "    self.sess = tf.Session(graph=self.graph)\n",
        "\n",
        "  def run(self, image):\n",
        "    \"\"\"Runs inference on a single image.\n",
        "\n",
        "    Args:\n",
        "      image: A PIL.Image object, raw input image.\n",
        "\n",
        "    Returns:\n",
        "      resized_image: RGB image resized from original input image.\n",
        "      seg_map: Segmentation map of `resized_image`.\n",
        "    \"\"\"\n",
        "    width, height = image.size\n",
        "    resize_ratio = 1.0 #* self.INPUT_SIZE / max(width, height)\n",
        "    target_size = image.size\n",
        "    resized_image = image.convert('RGB').resize(target_size, im.ANTIALIAS)\n",
        "    batch_seg_map = self.sess.run(\n",
        "        self.OUTPUT_TENSOR_NAME,\n",
        "        feed_dict={self.INPUT_TENSOR_NAME: [np.asarray(resized_image)]})\n",
        "    seg_map = batch_seg_map[0]\n",
        "    return resized_image, seg_map\n",
        "\n",
        "\n",
        "def create_pascal_label_colormap():\n",
        "  \"\"\"Creates a label colormap used in PASCAL VOC segmentation benchmark.\n",
        "\n",
        "  Returns:\n",
        "    A Colormap for visualizing segmentation results.\n",
        "  \"\"\"\n",
        "  colormap = np.zeros((256, 3), dtype=int)\n",
        "  ind = np.arange(256, dtype=int)\n",
        "\n",
        "  for shift in reversed(range(8)):\n",
        "    for channel in range(3):\n",
        "      colormap[:, channel] |= ((ind >> channel) & 1) << shift\n",
        "    ind >>= 3\n",
        "\n",
        "  return colormap\n",
        "\n",
        "\n",
        "def label_to_color_image(label):\n",
        "  \"\"\"Adds color defined by the dataset colormap to the label.\n",
        "\n",
        "  Args:\n",
        "    label: A 2D array with integer type, storing the segmentation label.\n",
        "\n",
        "  Returns:\n",
        "    result: A 2D array with floating type. The element of the array\n",
        "      is the color indexed by the corresponding element in the input label\n",
        "      to the PASCAL color map.\n",
        "\n",
        "  Raises:\n",
        "    ValueError: If label is not of rank 2 or its value is larger than color\n",
        "      map maximum entry.\n",
        "  \"\"\"\n",
        "  if label.ndim != 2:\n",
        "    raise ValueError('Expect 2-D input label')\n",
        "\n",
        "  colormap = create_pascal_label_colormap()\n",
        "\n",
        "  if np.max(label) >= len(colormap):\n",
        "    raise ValueError('label value too large.')\n",
        "\n",
        "  return colormap[label]\n",
        "\n",
        "\n",
        "def vis_segmentation(image, seg_map,img,loc):\n",
        "  \"\"\"Visualizes input image, segmentation map and overlay view.\"\"\"\n",
        "  seg_image = label_to_color_image(seg_map).astype(np.uint8)\n",
        "  np.save(os.path.join(loc,'image_raw_batch_seg_as_npy_0'+img),seg_image)\n",
        "\n",
        "LABEL_NAMES = np.asarray([\n",
        "    'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus',\n",
        "    'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike',\n",
        "    'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tv'\n",
        "])\n",
        "\n",
        "FULL_LABEL_MAP = np.arange(len(LABEL_NAMES)).reshape(len(LABEL_NAMES), 1)\n",
        "FULL_COLOR_MAP = label_to_color_image(FULL_LABEL_MAP)\n",
        "#for i in range(len(LABEL_NAMES)):\n",
        "#  print(LABEL_NAMES[i],end=' = ');\n",
        "#  print(FULL_COLOR_MAP[i])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YCzfjzh0WPow",
        "outputId": "f0fdb995-d662-4f42-bf23-09e8527753bc"
      },
      "source": [
        "MODEL_NAME = 'xception_coco_voctrainval'  # @param ['mobilenetv2_coco_voctrainaug', 'mobilenetv2_coco_voctrainval', 'xception_coco_voctrainaug', 'xception_coco_voctrainval']\n",
        "\n",
        "_DOWNLOAD_URL_PREFIX = 'http://download.tensorflow.org/models/'\n",
        "_MODEL_URLS = {\n",
        "    'mobilenetv2_coco_voctrainaug':\n",
        "        'deeplabv3_mnv2_pascal_train_aug_2018_01_29.tar.gz',\n",
        "    'mobilenetv2_coco_voctrainval':\n",
        "        'deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz',\n",
        "    'xception_coco_voctrainaug':\n",
        "        'deeplabv3_pascal_train_aug_2018_01_04.tar.gz',\n",
        "    'xception_coco_voctrainval':\n",
        "        'deeplabv3_pascal_trainval_2018_01_04.tar.gz',\n",
        "}\n",
        "_TARBALL_NAME = 'deeplab_model.tar.gz'\n",
        "\n",
        "model_dir = tempfile.mkdtemp()\n",
        "tf.gfile.MakeDirs(model_dir)\n",
        "\n",
        "download_path = os.path.join(model_dir, _TARBALL_NAME)\n",
        "print('downloading model, this might take a while...')\n",
        "urllib.request.urlretrieve(_DOWNLOAD_URL_PREFIX + _MODEL_URLS[MODEL_NAME],\n",
        "                   download_path)\n",
        "print('download completed! loading DeepLab model...')\n",
        "\n",
        "MODEL = DeepLabModel(download_path)\n",
        "print('model loaded successfully!')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "downloading model, this might take a while...\n",
            "download completed! loading DeepLab model...\n",
            "model loaded successfully!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "GhMPUiySWoVN"
      },
      "source": [
        "for i in range(100):\n",
        "  number = str(i//10) + str(i%10)\n",
        "  array = np.load('/content/content/image_raw_batch_histeq_as_npy/image_raw_batch_histeq_0' + number + '.npy')\n",
        "  original_im = im.fromarray(array)\n",
        "  resized_im, seg_map = MODEL.run(original_im)\n",
        "  # enter destination path for saving segmented images as npy files\n",
        "  vis_segmentation(original_im, seg_map,number,'/content/segmented_images_as_npy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4Oz3zpRv8mT9",
        "outputId": "4ff8dea1-bb63-4bfb-9b53-88f87f362726"
      },
      "source": [
        "A = np.load('/content/content/image_raw_batch_seg_as_npy/image_raw_batch_seg_as_npy_000.npy')\n",
        "A.shape\n",
        "A[0][0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 0, 0], dtype=uint8)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    }
  ]
}