Feature/enet image segmenter

ros/src/computing/perception/detection/packages/image_segmenter/CMakeLists.txt

@@ -0,0 +1,92 @@
+cmake_minimum_required(VERSION 2.8.12)
+project(image_segmenter)
+execute_process(
+  COMMAND rosversion -d
+  OUTPUT_VARIABLE ROS_VERSION
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+include(FindPkgConfig)
+
+if ("${ROS_VERSION}" MATCHES "(indigo|jade)")
+FIND_PACKAGE(catkin REQUIRED COMPONENTS
+  cv_bridge
+  image_transport
+  roscpp
+  sensor_msgs
+  std_msgs
+  autoware_msgs
+)
+elseif ("${ROS_VERSION}" MATCHES "(kinetic)")
+FIND_PACKAGE(catkin REQUIRED COMPONENTS
+  cv_bridge
+  image_transport
+  roscpp
+  std_msgs
+  autoware_msgs
+  sensor_msgs
+)
+endif()
+
+FIND_PACKAGE(CUDA)
+FIND_PACKAGE(OpenCV REQUIRED)
+
+EXECUTE_PROCESS(
+  COMMAND uname -m
+  OUTPUT_VARIABLE ARCHITECTURE
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+if ("${ROS_VERSION}" MATCHES "(indigo|jade)")
+catkin_package(
+  CATKIN_DEPENDS std_msgs geometry_msgs autoware_msgs
+)
+elseif ("${ROS_VERSION}" MATCHES "(kinetic)")
+catkin_package(
+  CATKIN_DEPENDS std_msgs geometry_msgs autoware_msgs
+)
+endif()
+###########
+## Build ##
+###########
+
+SET(CMAKE_CXX_FLAGS "-std=c++11 -O2 -g -Wall ${CMAKE_CXX_FLAGS}")
+
+INCLUDE_DIRECTORIES(
+  ${catkin_INCLUDE_DIRS}
+)
+
+
+#####ENET########
+##############################ENet's CAFFE FORK NEEDS TO BE PREVIOUSLY COMPILED####################
+set(ENET_CAFFE_PATH "$ENV{HOME}/ENet/caffe-enet/distribute")
+####################################################################################################
+if(EXISTS "${ENET_CAFFE_PATH}")
+
+  ADD_EXECUTABLE(image_segmenter_enet
+    src/image_segmenter_enet_node.cpp
+    src/image_segmenter_enet.cpp
+  )
+
+  TARGET_LINK_LIBRARIES(image_segmenter_enet
+    ${catkin_LIBRARIES}
+    ${OpenCV_LIBS}
+    ${CUDA_LIBRARIES}
+    ${CUDA_CUBLAS_LIBRARIES}
+    ${CUDA_curand_LIBRARY}
+    ${ENET_CAFFE_PATH}/lib/libcaffe.so
+    glog
+  )
+
+  TARGET_INCLUDE_DIRECTORIES(image_segmenter_enet PRIVATE
+    ${CUDA_INCLUDE_DIRS}
+    ${ENET_CAFFE_PATH}/include
+    include
+  )
+
+  ADD_DEPENDENCIES(image_segmenter_enet
+      ${catkin_EXPORTED_TARGETS}
+  )
+else()
+  message("'ENet/Caffe' is not installed. 'image_segmenter_enet' will not be built.")
+endif()

ros/src/computing/perception/detection/packages/image_segmenter/README.md

@@ -0,0 +1,30 @@
+# How to install Caffe for ENet
+
+1. Complete the [pre-requisites](http://caffe.berkeleyvision.org/install_apt.html) 
+
+2. Clone ENEt Caffe fork in your home directory
+```
+% git clone --recursive https://github.com/TimoSaemann/ENet.git
+```
+
+3. Follow the authors' instruction to complete the requisites to compile as shown in
+https://github.com/TimoSaemann/ENet/tree/master/Tutorial
+
+4. Compile ENet fork of Caffe using Make (http://caffe.berkeleyvision.org/installation.html#compilation)
+*Don't use CMake to compile, manually modify Makefile.config*
+```
+make && make distribute
+```
+
+6. Download pretrained models as pointed in https://github.com/TimoSaemann/ENet/tree/master/Tutorial#kick-start or use your own.
+
+If you didn't install ENet Caffe in `ENet` inside your home, modify the ENet's CMakeFiles and point them to your directories.
+
+Once compiled, run from the terminal, or launch from RunTimeManager
+
+```
+% roslaunch image_segmenter image_segmenter_enet.launch 
+```
+Remember to modify the launch file located inside 
+`computing/perception/detection/packages/enet_image_segmenter/launch/enet_image_segmenter.launch`
+and point the network, pretrained models and LUT file to your paths.

ros/src/computing/perception/detection/packages/image_segmenter/include/image_segmenter_enet.hpp

@@ -0,0 +1,52 @@
+/*
+ * image_segmenter_enet.hpp
+ *
+ *  Created on: Aug 23, 2017
+ *      Author: ne0
+ */
+
+#ifndef ENET_IMAGE_SEGMENTER_HPP_
+#define ENET_IMAGE_SEGMENTER_HPP_
+
+#include <caffe/caffe.hpp>
+
+#include <opencv2/core/core.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include <algorithm>
+#include <iosfwd>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+using namespace caffe;
+using std::string;
+
+class ENetSegmenter
+{
+public:
+	ENetSegmenter(const string& in_model_file, const string& in_trained_file, const string& in_lookuptable_file);
+
+	void Predict(const cv::Mat& in_img, cv::Mat& out_segmented);
+
+private:
+	void SetMean(const string& in_mean_file);
+
+	void WrapInputLayer(std::vector<cv::Mat>* in_input_channels);
+
+	void Preprocess(const cv::Mat& img, std::vector<cv::Mat>* in_input_channels);
+
+	cv::Mat Visualization(cv::Mat in_prediction_map, string in_lookuptable_file);
+
+private:
+	shared_ptr<Net<float> > 	net_;
+	cv::Size 					input_geometry_;
+	int 						num_channels_;
+	string 						lookuptable_file_;
+	cv::Scalar 					pixel_mean_;
+
+};
+
+#endif /* ENET_IMAGE_SEGMENTER_HPP_ */

ros/src/computing/perception/detection/packages/image_segmenter/launch/image_segmenter_enet.launch

@@ -0,0 +1,18 @@
+<launch>
+	<!-- arguments list -->
+	<arg name="network_definition_file" default="$(env HOME)/ENet/prototxts/enet_deploy_final.prototxt"/>
+	<arg name="pretrained_model_file" default="$(env HOME)/ENet/enet_weights_zoo/cityscapes_weights.caffemodel"/>
+	<arg name="lookuptable_file" default="$(env HOME)/ENet/scripts/cityscapes19.png"/> 
+
+	<arg name="camera_id" default="/" />
+	<arg name="image_src" default="/image_raw"/>
+
+	<!-- ENET -->
+	<node pkg="image_segmenter" name="image_segmenter_enet" type="image_segmenter_enet" output="screen">
+		<param name="network_definition_file" type="str" value="$(arg network_definition_file)"/>
+		<param name="pretrained_model_file" type="str" value="$(arg pretrained_model_file)"/>
+		<param name="lookuptable_file" type="str" value="$(arg lookuptable_file)"/>
+		<param name="image_raw_node" type="str" value="$(arg camera_id)$(arg image_src)"/>
+	</node>
+
+</launch>

ros/src/computing/perception/detection/packages/image_segmenter/package.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<package>
+  <name>image_segmenter</name>
+  <version>1.0.0</version>
+  <description>Image Segmentation</description>
+  <maintainer email="abrahammonrroy@yahoo.com">Abraham Monrroy</maintainer>
+  <license>BSD</license>
+  <buildtool_depend>catkin</buildtool_depend>
+  <build_depend>std_msgs</build_depend>
+  <build_depend>geometry_msgs</build_depend>
+  <build_depend>autoware_msgs</build_depend>
+  <run_depend>std_msgs</run_depend>
+  <run_depend>geometry_msgs</run_depend>
+  <run_depend>autoware_msgs</run_depend>
+  <export>
+  </export>
+</package>

ros/src/computing/perception/detection/packages/image_segmenter/src/image_segmenter_enet.cpp

@@ -0,0 +1,141 @@
+/*
+ * image_segmenter_enet.cpp
+ *
+ *  Created on: Aug 23, 2017
+ *      Author: amc
+ */
+
+#include "image_segmenter_enet.hpp"
+
+ENetSegmenter::ENetSegmenter(const string& in_model_file,
+		const string& in_trained_file,
+		const string& in_lookuptable_file)
+{
+
+	Caffe::set_mode(Caffe::GPU);
+
+	/* Load the network. */
+	net_.reset(new Net<float>(in_model_file, TEST));
+	net_->CopyTrainedLayersFrom(in_trained_file);
+
+	CHECK_EQ(net_->num_inputs(), 1)<< "Network should have exactly one input.";
+	CHECK_EQ(net_->num_outputs(), 1)<< "Network should have exactly one output.";
+
+	Blob<float>* input_layer = net_->input_blobs()[0];
+	num_channels_ = input_layer->channels();
+	CHECK(num_channels_ == 3 || num_channels_ == 1) << "Input layer should have 1 or 3 channels.";
+	input_geometry_ = cv::Size(input_layer->width(), input_layer->height());
+
+	lookuptable_file_ = in_lookuptable_file;
+
+	pixel_mean_		= cv::Scalar(102.9801, 115.9465, 122.7717);
+}
+
+void ENetSegmenter::Predict(const cv::Mat& in_image_mat, cv::Mat& out_segmented)
+{
+	Blob<float>* input_layer = net_->input_blobs()[0];
+	input_layer->Reshape(1, num_channels_, input_geometry_.height,
+			input_geometry_.width);
+	/* Forward dimension change to all layers. */
+	net_->Reshape();
+
+	std::vector<cv::Mat> input_channels;
+	WrapInputLayer(&input_channels);
+
+	Preprocess(in_image_mat, &input_channels);
+
+	net_->Forward();
+
+	/* Copy the output layer to a std::vector */
+	Blob<float>* output_layer = net_->output_blobs()[0];
+
+	int width = output_layer->width();
+	int height = output_layer->height();
+	int channels = output_layer->channels();
+	int num = output_layer->num();
+
+	// compute argmax
+	cv::Mat class_each_row(channels, width * height, CV_32FC1,
+			const_cast<float *>(output_layer->cpu_data()));
+	class_each_row = class_each_row.t(); // transpose to make each row with all probabilities
+	cv::Point maxId;    // point [x,y] values for index of max
+	double maxValue;    // the holy max value itself
+	cv::Mat prediction_map(height, width, CV_8UC1);
+	for (int i = 0; i < class_each_row.rows; i++)
+	{
+		minMaxLoc(class_each_row.row(i), 0, &maxValue, 0, &maxId);
+		prediction_map.at<uchar>(i) = maxId.x;
+	}
+
+	out_segmented = Visualization(prediction_map, lookuptable_file_);
+
+	cv::resize(out_segmented, out_segmented, cv::Size(in_image_mat.cols, in_image_mat.rows));
+
+
+}
+
+cv::Mat ENetSegmenter::Visualization(cv::Mat in_prediction_map, string in_lookuptable_file)
+{
+
+	cv::cvtColor(in_prediction_map.clone(), in_prediction_map, CV_GRAY2BGR);
+	cv::Mat label_colours = cv::imread(in_lookuptable_file, 1);
+	cv::cvtColor(label_colours, label_colours, CV_RGB2BGR);
+	cv::Mat output_image;
+	LUT(in_prediction_map, label_colours, output_image);
+
+	return output_image;
+}
+
+void ENetSegmenter::WrapInputLayer(std::vector<cv::Mat>* in_input_channels)
+{
+	Blob<float>* input_layer = net_->input_blobs()[0];
+
+	int width = input_layer->width();
+	int height = input_layer->height();
+	float* input_data = input_layer->mutable_cpu_data();
+	for (int i = 0; i < input_layer->channels(); ++i)
+	{
+		cv::Mat channel(height, width, CV_32FC1, input_data);
+		in_input_channels->push_back(channel);
+		input_data += width * height;
+	}
+}
+
+void ENetSegmenter::Preprocess(const cv::Mat& in_image_mat,
+		std::vector<cv::Mat>* in_input_channels)
+{
+	/* Convert the input image to the input image format of the network. */
+	cv::Mat sample;
+	if (in_image_mat.channels() == 3 && num_channels_ == 1)
+		cv::cvtColor(in_image_mat, sample, cv::COLOR_BGR2GRAY);
+	else if (in_image_mat.channels() == 4 && num_channels_ == 1)
+		cv::cvtColor(in_image_mat, sample, cv::COLOR_BGRA2GRAY);
+	else if (in_image_mat.channels() == 4 && num_channels_ == 3)
+		cv::cvtColor(in_image_mat, sample, cv::COLOR_BGRA2BGR);
+	else if (in_image_mat.channels() == 1 && num_channels_ == 3)
+		cv::cvtColor(in_image_mat, sample, cv::COLOR_GRAY2BGR);
+	else
+		sample = in_image_mat;
+
+	cv::Mat sample_resized;
+	if (sample.size() != input_geometry_)
+		cv::resize(sample, sample_resized, input_geometry_);
+	else
+		sample_resized = sample;
+
+	cv::Mat sample_float;
+	if (num_channels_ == 3)
+	{
+		sample_resized.convertTo(sample_float, CV_32FC3);
+	}
+	else
+	{
+		sample_resized.convertTo(sample_float, CV_32FC1);
+	}
+
+	cv::split(sample_float, *in_input_channels);
+
+	CHECK(reinterpret_cast<float*>(in_input_channels->at(0).data)
+			== net_->input_blobs()[0]->cpu_data())
+															<< "Input channels are not wrapping the input layer of the network.";
+}