Fix segmentation

.ci_local_test/ros2_openvino_toolkit_test/test_cases/run.sh

@@ -16,7 +16,7 @@ apt-get update
 # apt-get install -y ros-$ros2_branch-diagnostic-updater
 apt-get install python3-defusedxml
 apt-get install -y python3-pip
-pip3 install XTestRunner
+pip3 install XTestRunner==1.5.0
 
 cd /root/test_cases && ./ros2_openvino_tool_model_download.sh
 mkdir -p /root/test_cases/log

openvino_wrapper_lib/include/openvino_wrapper_lib/models/attributes/base_attribute.hpp

@@ -160,11 +160,21 @@ class ModelAttribute
     attr_.input_height = height;
   }
 
+  inline int getInputHeight() const
+  {
+    return attr_.input_height;
+  }
+
   inline void setInputWidth(const int width)
   {
     attr_.input_width = width;
   }
 
+  inline int getInputWidth() const
+  {
+    return attr_.input_width;
+  }
+
   inline void setMaxProposalCount(const int max)
   {
     attr_.max_proposal_count = max;
@@ -195,4 +205,4 @@ class SSDModelAttr : public ModelAttribute
 
 }  // namespace Models
 
-#endif  // OPENVINO_WRAPPER_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_
+#endif  // OPENVINO_WRAPPER_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_

openvino_wrapper_lib/src/models/object_segmentation_maskrcnn_model.cpp

@@ -86,10 +86,10 @@ bool Models::ObjectSegmentationMaskrcnnModel::matToBlob(
   ov::Shape input_shape = input_tensor.get_shape();
 
   OPENVINO_ASSERT(input_shape.size() == 4);
-  // For frozen graph model:
-  const size_t width = input_shape[3];
-  const size_t height = input_shape[2];
-  const size_t channels = input_shape[1];
+  // For frozen graph model: layout= "NHWC"
+  const size_t width = input_shape[2];
+  const size_t height = input_shape[1];
+  const size_t channels = input_shape[3];
 
   slog::debug <<"width is:"<< width << slog::endl;
   slog::debug <<"height is:"<< height << slog::endl;
@@ -101,6 +101,14 @@ bool Models::ObjectSegmentationMaskrcnnModel::matToBlob(
      throw std::runtime_error("The number of channels for net input and image must match");
   }
 
+#if 1
+  //input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, resized_image.data);
+  //engine->getRequest().set_tensor(input_tensor_name_, input_tensor);
+  unsigned char* data = input_tensor.data<unsigned char>();
+  cv::Size size = {(int)width, (int)height};
+  cv::Mat resized_image(size, CV_8UC3, data);
+  cv::resize(orig_image, resized_image, size);
+#else
   const auto input_data = input_tensor.data<unsigned char>();
   cv::Mat resized_image(orig_image);
   if (static_cast<int>(width) != orig_image.size().width ||
@@ -127,6 +135,7 @@ bool Models::ObjectSegmentationMaskrcnnModel::matToBlob(
   } else {
       throw std::runtime_error("Unsupported number of channels");
   }
+#endif
 
   return true;
 }

openvino_wrapper_lib/src/models/object_segmentation_model.cpp

@@ -78,26 +78,49 @@ bool Models::ObjectSegmentationModel::matToBlob(
     slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl;
     return false;
   }
-
-  size_t channels = orig_image.channels();
-  size_t height = orig_image.size().height;
-  size_t width = orig_image.size().width;
-
-  size_t strideH = orig_image.step.buf[0];
-  size_t strideW = orig_image.step.buf[1];
-
-  bool is_dense =
-      strideW == channels &&
-      strideH == channels * width;
-
-  if (!is_dense){
-    slog::err << "Doesn't support conversion from not dense cv::Mat." << slog::endl;
-    return false;
+#if 1
+  const size_t width = getInputWidth();
+  const size_t height = getInputHeight();
+  const size_t channels = 3;
+  slog::debug <<"width is:"<< width << slog::endl;
+  slog::debug <<"height is:"<< height << slog::endl;
+
+  if (orig_image.cols != width || orig_image.rows != height){
+    cv::Size size = {(int)width, (int)height};
+    cv::Mat resized_image(size, CV_8UC3);
+    cv::resize(orig_image, resized_image, size);
+    ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, resized_image.data);
+    engine->getRequest().set_tensor(input_tensor_name_, input_tensor);
+  } else {
+    ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, orig_image.data);
+    engine->getRequest().set_tensor(input_tensor_name_, input_tensor);
   }
-
-  ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, orig_image.data);
-  engine->getRequest().set_tensor(input_tensor_name_, input_tensor);
-
+#else
+  ov::InferRequest infer_request = engine->getRequest();
+  ov::Tensor input_tensor = infer_request.get_tensor(getInputName("input"));
+  ov::Shape input_shape = input_tensor.get_shape();
+
+  OPENVINO_ASSERT(input_shape.size() == 4);
+  // For frozen graph model:
+  const size_t width = input_shape[2];
+  const size_t height = input_shape[1];
+  const size_t channels = input_shape[3];
+
+  slog::debug <<"width is:"<< width << slog::endl;
+  slog::debug <<"height is:"<< height << slog::endl;
+  slog::debug <<"channels is:"<< channels << slog::endl;
+  slog::debug <<"origin channels is:"<< orig_image.channels() << slog::endl;
+  slog::debug <<"input shape is:"<< input_shape << slog::endl;
+
+  if (static_cast<size_t>(orig_image.channels()) != channels) {
+     throw std::runtime_error("The number of channels for net input and image must match");
+  }
+
+  unsigned char* data = input_tensor.data<unsigned char>();
+  cv::Size size = {(int)width, (int)height};
+  cv::Mat resized_image(size, CV_8UC3, data);
+  cv::resize(orig_image, resized_image, size);
+#endif
   return true;
 }
 
@@ -126,11 +149,15 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(
   ov::Layout tensor_layout = ov::Layout("NHWC");
   ov::Layout expect_layout = ov::Layout("NCHW");
   ov::Shape input_shape = model->input().get_shape();
-  if (input_shape[1] == 3)
+  if (input_shape[1] == 3){
     expect_layout = ov::Layout("NCHW");
-  else if (input_shape[3] == 3)
+    setInputWidth(input_shape[3]);
+    setInputHeight(input_shape[2]);
+  } else if (input_shape[3] == 3){
     expect_layout = ov::Layout("NHWC");
-  else
+    setInputWidth(input_shape[2]);
+    setInputHeight(input_shape[1]);
+  } else
     slog::warn << "unexpect input shape " << input_shape << slog::endl;
 
   input_info.tensor().
@@ -167,16 +194,22 @@ bool Models::ObjectSegmentationModel::updateLayerProperty(
   auto& outSizeVector = data.get_shape();
   int outChannels, outHeight, outWidth;
   slog::debug << "output size vector " << outSizeVector.size() << slog::endl;
+  ov::Layout outputLayout("");
   switch(outSizeVector.size()){
     case 3:
-      outChannels = 0;
-      outHeight = outSizeVector[1];
-      outWidth = outSizeVector[2];
+      outputLayout = "CHW";
+      outChannels = 1;
+      outHeight = static_cast<int>(outSizeVector[ov::layout::height_idx(outputLayout)]);
+      outWidth = static_cast<int>(outSizeVector[ov::layout::width_idx(outputLayout)]);
       break;
     case 4:
-      outChannels = outSizeVector[1];
-      outHeight = outSizeVector[2];
-      outWidth = outSizeVector[3];
+      //outChannels = outSizeVector[1];
+      //outHeight = outSizeVector[2];
+      //outWidth = outSizeVector[3];
+      outputLayout = "NCHW";
+      outChannels = static_cast<int>(outSizeVector[ov::layout::channels_idx(outputLayout)]);
+      outHeight = static_cast<int>(outSizeVector[ov::layout::height_idx(outputLayout)]);
+      outWidth = static_cast<int>(outSizeVector[ov::layout::width_idx(outputLayout)]);
       break;
     default:
       throw std::runtime_error("Unexpected output blob shape. Only 4D and 3D output blobs are"

openvino_wrapper_lib/src/outputs/image_window_output.cpp

@@ -143,39 +143,21 @@ void Outputs::ImageWindowOutput::accept(
   }
 }
 
+
 void Outputs::ImageWindowOutput::mergeMask(
   const std::vector<openvino_wrapper_lib::ObjectSegmentationResult> & results)
 {
-  std::map<std::string, int> class_color;
-  for (unsigned i = 0; i < results.size(); i++) {
-    std::string class_label = results[i].getLabel();
-    if (class_color.find(class_label) == class_color.end()) {
-      class_color[class_label] = class_color.size();
-    }
-    auto & color = colors_[class_color[class_label] % colors_.size() ];
     const float alpha = 0.7f;
-    const float MASK_THRESHOLD = 0.5;
-
-    cv::Rect location = results[i].getLocation();
-    cv::Mat roi_img = frame_(location);
-    cv::Mat mask = results[i].getMask();
-    cv::Mat colored_mask(location.height, location.width, frame_.type(),
-		   cv::Scalar(color[2], color[1], color[0]) );
-    roi_img.copyTo(colored_mask, mask <= MASK_THRESHOLD);
-
-/**
-    for (int h = 0; h < mask.size().height; ++h) {
-      for (int w = 0; w < mask.size().width; ++w) {
-        for (int ch = 0; ch < colored_mask.channels(); ++ch) {
-          colored_mask.at<cv::Vec3b>(h, w)[ch] = mask.at<float>(h, w) > MASK_THRESHOLD ?
-            255 * color[ch] :
-            roi_img.at<cv::Vec3b>(h, w)[ch];
-        }
-      }
+    //const float MASK_THRESHOLD = 0.5;
+    //only for merged mask mat got from modles::fetchResults()
+    for (unsigned i=0; i<results.size(); i++){
+      cv::Rect location = results[i].getLocation();
+      slog::debug << "Rect:" << location << slog::endl;
+      slog::debug << " Frame Size: " << frame_.size() << slog::endl;
+      cv::Mat mask = results[i].getMask();
+      cv::resize(mask, mask, frame_.size());
+      cv::addWeighted(mask, alpha, frame_, 1.0f - alpha, 0.0f, frame_);
     }
-*/
-    cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img);
-  }
 }
 
 void Outputs::ImageWindowOutput::accept(
@@ -184,23 +166,20 @@ void Outputs::ImageWindowOutput::accept(
   for (unsigned i = 0; i < results.size(); i++) {
     cv::Rect result_rect = results[i].getLocation();
     unsigned target_index = findOutput(result_rect);
-    outputs_[target_index].rect = result_rect;
+
     auto fd_conf = results[i].getConfidence();
-    if (fd_conf >= 0) {
+    if (fd_conf > 0) {
+      outputs_[target_index].rect = result_rect;
       std::ostringstream ostream;
       ostream << "[" << std::fixed << std::setprecision(3) << fd_conf << "]";
       outputs_[target_index].desc += ostream.str();
+      auto label = results[i].getLabel();
+      outputs_[target_index].desc += "[" + label + "]";
     }
-    auto label = results[i].getLabel();
-    outputs_[target_index].desc += "[" + label + "]";
   }
   mergeMask(results);
 }
 
-
-
-
-
 void Outputs::ImageWindowOutput::mergeMask(
   const std::vector<openvino_wrapper_lib::ObjectSegmentationMaskrcnnResult> & results)
 {

sample/param/pipeline_segmentation_maskrcnn.yaml

@@ -3,7 +3,7 @@ Pipelines:
   inputs: [StandardCamera]
   infers:
     - name: ObjectSegmentationMaskrcnn
-      model: /opt/openvino_toolkit/models/convert/public/mask_rcnn_inception_v2_coco_2018_01_28/OUT/frozen_inference_graph.xml
+      model: /opt/openvino_toolkit/models/public/mask_rcnn_inception_resnet_v2_atrous_coco/FP16/mask_rcnn_inception_resnet_v2_atrous_coco.xml
       engine: CPU #"HETERO:CPU,GPU,MYRIAD"
       label: to/be/set/xxx.labels
       batch: 1