From 8f7380408b8c50c7ecd281aa72ddfe5b86621960 Mon Sep 17 00:00:00 2001 From: "Liu, Wei Zhi" Date: Fri, 10 Mar 2023 11:43:27 +0800 Subject: [PATCH 1/5] update code to align with new version of maskrcnn segmentation model (layout=NHWC) Signed-off-by: Liu, Wei Zhi --- .../object_segmentation_maskrcnn_model.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/openvino_wrapper_lib/src/models/object_segmentation_maskrcnn_model.cpp b/openvino_wrapper_lib/src/models/object_segmentation_maskrcnn_model.cpp index d0d127ce..d08b5a57 100644 --- a/openvino_wrapper_lib/src/models/object_segmentation_maskrcnn_model.cpp +++ b/openvino_wrapper_lib/src/models/object_segmentation_maskrcnn_model.cpp @@ -86,10 +86,10 @@ bool Models::ObjectSegmentationMaskrcnnModel::matToBlob( ov::Shape input_shape = input_tensor.get_shape(); OPENVINO_ASSERT(input_shape.size() == 4); - // For frozen graph model: - const size_t width = input_shape[3]; - const size_t height = input_shape[2]; - const size_t channels = input_shape[1]; + // For frozen graph model: layout= "NHWC" + const size_t width = input_shape[2]; + const size_t height = input_shape[1]; + const size_t channels = input_shape[3]; slog::debug <<"width is:"<< width << slog::endl; slog::debug <<"height is:"<< height << slog::endl; @@ -101,6 +101,14 @@ bool Models::ObjectSegmentationMaskrcnnModel::matToBlob( throw std::runtime_error("The number of channels for net input and image must match"); } +#if 1 + //input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, resized_image.data); + //engine->getRequest().set_tensor(input_tensor_name_, input_tensor); + unsigned char* data = input_tensor.data(); + cv::Size size = {(int)width, (int)height}; + cv::Mat resized_image(size, CV_8UC3, data); + cv::resize(orig_image, resized_image, size); +#else const auto input_data = input_tensor.data(); cv::Mat resized_image(orig_image); if (static_cast(width) != orig_image.size().width || @@ -127,6 +135,7 @@ bool Models::ObjectSegmentationMaskrcnnModel::matToBlob( } else { throw std::runtime_error("Unsupported number of channels"); } +#endif return true; } From bf9a124ebc79442700424420bd68d9d0f2e3e978 Mon Sep 17 00:00:00 2001 From: "Liu, Wei Zhi" Date: Fri, 10 Mar 2023 12:03:19 +0800 Subject: [PATCH 2/5] update code to align with new version of deeplabv3 segmentation model. Signed-off-by: Liu, Wei Zhi --- .../models/attributes/base_attribute.hpp | 12 ++- .../src/models/object_segmentation_model.cpp | 89 +++++++++++++------ .../src/outputs/image_window_output.cpp | 44 +++------ 3 files changed, 82 insertions(+), 63 deletions(-) diff --git a/openvino_wrapper_lib/include/openvino_wrapper_lib/models/attributes/base_attribute.hpp b/openvino_wrapper_lib/include/openvino_wrapper_lib/models/attributes/base_attribute.hpp index e439e9f0..7f36c061 100644 --- a/openvino_wrapper_lib/include/openvino_wrapper_lib/models/attributes/base_attribute.hpp +++ b/openvino_wrapper_lib/include/openvino_wrapper_lib/models/attributes/base_attribute.hpp @@ -160,11 +160,21 @@ class ModelAttribute attr_.input_height = height; } + inline int getInputHeight() const + { + return attr_.input_height; + } + inline void setInputWidth(const int width) { attr_.input_width = width; } + inline int getInputWidth() const + { + return attr_.input_width; + } + inline void setMaxProposalCount(const int max) { attr_.max_proposal_count = max; @@ -195,4 +205,4 @@ class SSDModelAttr : public ModelAttribute } // namespace Models -#endif // OPENVINO_WRAPPER_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_ \ No newline at end of file +#endif // OPENVINO_WRAPPER_LIB__MODELS__ATTRIBUTES_BASE_ATTRIBUTE_HPP_ diff --git a/openvino_wrapper_lib/src/models/object_segmentation_model.cpp b/openvino_wrapper_lib/src/models/object_segmentation_model.cpp index 31cd4c78..1d076757 100644 --- a/openvino_wrapper_lib/src/models/object_segmentation_model.cpp +++ b/openvino_wrapper_lib/src/models/object_segmentation_model.cpp @@ -78,26 +78,49 @@ bool Models::ObjectSegmentationModel::matToBlob( slog::err << "A frame is trying to be enqueued in a NULL Engine." << slog::endl; return false; } - - size_t channels = orig_image.channels(); - size_t height = orig_image.size().height; - size_t width = orig_image.size().width; - - size_t strideH = orig_image.step.buf[0]; - size_t strideW = orig_image.step.buf[1]; - - bool is_dense = - strideW == channels && - strideH == channels * width; - - if (!is_dense){ - slog::err << "Doesn't support conversion from not dense cv::Mat." << slog::endl; - return false; +#if 1 + const size_t width = getInputWidth(); + const size_t height = getInputHeight(); + const size_t channels = 3; + slog::debug <<"width is:"<< width << slog::endl; + slog::debug <<"height is:"<< height << slog::endl; + + if (orig_image.cols != width || orig_image.rows != height){ + cv::Size size = {(int)width, (int)height}; + cv::Mat resized_image(size, CV_8UC3); + cv::resize(orig_image, resized_image, size); + ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, resized_image.data); + engine->getRequest().set_tensor(input_tensor_name_, input_tensor); + } else { + ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, orig_image.data); + engine->getRequest().set_tensor(input_tensor_name_, input_tensor); } - - ov::Tensor input_tensor = ov::Tensor(ov::element::u8, {1, height, width, channels}, orig_image.data); - engine->getRequest().set_tensor(input_tensor_name_, input_tensor); - +#else + ov::InferRequest infer_request = engine->getRequest(); + ov::Tensor input_tensor = infer_request.get_tensor(getInputName("input")); + ov::Shape input_shape = input_tensor.get_shape(); + + OPENVINO_ASSERT(input_shape.size() == 4); + // For frozen graph model: + const size_t width = input_shape[2]; + const size_t height = input_shape[1]; + const size_t channels = input_shape[3]; + + slog::debug <<"width is:"<< width << slog::endl; + slog::debug <<"height is:"<< height << slog::endl; + slog::debug <<"channels is:"<< channels << slog::endl; + slog::debug <<"origin channels is:"<< orig_image.channels() << slog::endl; + slog::debug <<"input shape is:"<< input_shape << slog::endl; + + if (static_cast(orig_image.channels()) != channels) { + throw std::runtime_error("The number of channels for net input and image must match"); + } + + unsigned char* data = input_tensor.data(); + cv::Size size = {(int)width, (int)height}; + cv::Mat resized_image(size, CV_8UC3, data); + cv::resize(orig_image, resized_image, size); +#endif return true; } @@ -126,11 +149,15 @@ bool Models::ObjectSegmentationModel::updateLayerProperty( ov::Layout tensor_layout = ov::Layout("NHWC"); ov::Layout expect_layout = ov::Layout("NCHW"); ov::Shape input_shape = model->input().get_shape(); - if (input_shape[1] == 3) + if (input_shape[1] == 3){ expect_layout = ov::Layout("NCHW"); - else if (input_shape[3] == 3) + setInputWidth(input_shape[3]); + setInputHeight(input_shape[2]); + } else if (input_shape[3] == 3){ expect_layout = ov::Layout("NHWC"); - else + setInputWidth(input_shape[2]); + setInputHeight(input_shape[1]); + } else slog::warn << "unexpect input shape " << input_shape << slog::endl; input_info.tensor(). @@ -167,16 +194,22 @@ bool Models::ObjectSegmentationModel::updateLayerProperty( auto& outSizeVector = data.get_shape(); int outChannels, outHeight, outWidth; slog::debug << "output size vector " << outSizeVector.size() << slog::endl; + ov::Layout outputLayout(""); switch(outSizeVector.size()){ case 3: - outChannels = 0; - outHeight = outSizeVector[1]; - outWidth = outSizeVector[2]; + outputLayout = "CHW"; + outChannels = 1; + outHeight = static_cast(outSizeVector[ov::layout::height_idx(outputLayout)]); + outWidth = static_cast(outSizeVector[ov::layout::width_idx(outputLayout)]); break; case 4: - outChannels = outSizeVector[1]; - outHeight = outSizeVector[2]; - outWidth = outSizeVector[3]; + //outChannels = outSizeVector[1]; + //outHeight = outSizeVector[2]; + //outWidth = outSizeVector[3]; + outputLayout = "NCHW"; + outChannels = static_cast(outSizeVector[ov::layout::channels_idx(outputLayout)]); + outHeight = static_cast(outSizeVector[ov::layout::height_idx(outputLayout)]); + outWidth = static_cast(outSizeVector[ov::layout::width_idx(outputLayout)]); break; default: throw std::runtime_error("Unexpected output blob shape. Only 4D and 3D output blobs are" diff --git a/openvino_wrapper_lib/src/outputs/image_window_output.cpp b/openvino_wrapper_lib/src/outputs/image_window_output.cpp index 1cb37940..f93c123f 100644 --- a/openvino_wrapper_lib/src/outputs/image_window_output.cpp +++ b/openvino_wrapper_lib/src/outputs/image_window_output.cpp @@ -146,36 +146,15 @@ void Outputs::ImageWindowOutput::accept( void Outputs::ImageWindowOutput::mergeMask( const std::vector & results) { - std::map class_color; - for (unsigned i = 0; i < results.size(); i++) { - std::string class_label = results[i].getLabel(); - if (class_color.find(class_label) == class_color.end()) { - class_color[class_label] = class_color.size(); - } - auto & color = colors_[class_color[class_label] % colors_.size() ]; const float alpha = 0.7f; - const float MASK_THRESHOLD = 0.5; + //const float MASK_THRESHOLD = 0.5; cv::Rect location = results[i].getLocation(); - cv::Mat roi_img = frame_(location); + slog::debug << "Rect:" << location << slog::endl; + slog::debug << " Frame Size: " << frame_.size() << slog::endl; cv::Mat mask = results[i].getMask(); - cv::Mat colored_mask(location.height, location.width, frame_.type(), - cv::Scalar(color[2], color[1], color[0]) ); - roi_img.copyTo(colored_mask, mask <= MASK_THRESHOLD); - -/** - for (int h = 0; h < mask.size().height; ++h) { - for (int w = 0; w < mask.size().width; ++w) { - for (int ch = 0; ch < colored_mask.channels(); ++ch) { - colored_mask.at(h, w)[ch] = mask.at(h, w) > MASK_THRESHOLD ? - 255 * color[ch] : - roi_img.at(h, w)[ch]; - } - } - } -*/ - cv::addWeighted(colored_mask, alpha, roi_img, 1.0f - alpha, 0.0f, roi_img); - } + cv::resize(mask, mask, frame_.size()); + cv::addWeighted(mask, alpha, frame_, 1.0f - alpha, 0.0f, frame_); } void Outputs::ImageWindowOutput::accept( @@ -184,23 +163,20 @@ void Outputs::ImageWindowOutput::accept( for (unsigned i = 0; i < results.size(); i++) { cv::Rect result_rect = results[i].getLocation(); unsigned target_index = findOutput(result_rect); - outputs_[target_index].rect = result_rect; + auto fd_conf = results[i].getConfidence(); - if (fd_conf >= 0) { + if (fd_conf > 0) { + outputs_[target_index].rect = result_rect; std::ostringstream ostream; ostream << "[" << std::fixed << std::setprecision(3) << fd_conf << "]"; outputs_[target_index].desc += ostream.str(); + auto label = results[i].getLabel(); + outputs_[target_index].desc += "[" + label + "]"; } - auto label = results[i].getLabel(); - outputs_[target_index].desc += "[" + label + "]"; } mergeMask(results); } - - - - void Outputs::ImageWindowOutput::mergeMask( const std::vector & results) { From c3e48a1114bb9e0e66b38376f81eefa9473bcd28 Mon Sep 17 00:00:00 2001 From: huangjiafengx Date: Fri, 10 Mar 2023 13:42:14 +0800 Subject: [PATCH 3/5] Update run.sh (#270) --- .ci_local_test/ros2_openvino_toolkit_test/test_cases/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_local_test/ros2_openvino_toolkit_test/test_cases/run.sh b/.ci_local_test/ros2_openvino_toolkit_test/test_cases/run.sh index eb362cc5..e3093d3e 100755 --- a/.ci_local_test/ros2_openvino_toolkit_test/test_cases/run.sh +++ b/.ci_local_test/ros2_openvino_toolkit_test/test_cases/run.sh @@ -16,7 +16,7 @@ apt-get update # apt-get install -y ros-$ros2_branch-diagnostic-updater apt-get install python3-defusedxml apt-get install -y python3-pip -pip3 install XTestRunner +pip3 install XTestRunner==1.5.0 cd /root/test_cases && ./ros2_openvino_tool_model_download.sh mkdir -p /root/test_cases/log From d3bb8d9a8f3e26ef93cb2b6a89980e5475900e71 Mon Sep 17 00:00:00 2001 From: "Liu, Wei Zhi" Date: Fri, 10 Mar 2023 13:42:52 +0800 Subject: [PATCH 4/5] fix build error. Signed-off-by: Liu, Wei Zhi --- .../src/outputs/image_window_output.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/openvino_wrapper_lib/src/outputs/image_window_output.cpp b/openvino_wrapper_lib/src/outputs/image_window_output.cpp index f93c123f..0f2b93e2 100644 --- a/openvino_wrapper_lib/src/outputs/image_window_output.cpp +++ b/openvino_wrapper_lib/src/outputs/image_window_output.cpp @@ -143,18 +143,21 @@ void Outputs::ImageWindowOutput::accept( } } + void Outputs::ImageWindowOutput::mergeMask( const std::vector & results) { const float alpha = 0.7f; //const float MASK_THRESHOLD = 0.5; - - cv::Rect location = results[i].getLocation(); - slog::debug << "Rect:" << location << slog::endl; - slog::debug << " Frame Size: " << frame_.size() << slog::endl; - cv::Mat mask = results[i].getMask(); - cv::resize(mask, mask, frame_.size()); - cv::addWeighted(mask, alpha, frame_, 1.0f - alpha, 0.0f, frame_); + //only for merged mask mat got from modles::fetchResults() + for (unsigned i=0; i Date: Fri, 10 Mar 2023 14:30:15 +0800 Subject: [PATCH 5/5] update yaml file to use new version of maskrcnn segmentation model. Signed-off-by: Liu, Wei Zhi --- sample/param/pipeline_segmentation_maskrcnn.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample/param/pipeline_segmentation_maskrcnn.yaml b/sample/param/pipeline_segmentation_maskrcnn.yaml index d5b8671a..fa47f088 100644 --- a/sample/param/pipeline_segmentation_maskrcnn.yaml +++ b/sample/param/pipeline_segmentation_maskrcnn.yaml @@ -3,7 +3,7 @@ Pipelines: inputs: [StandardCamera] infers: - name: ObjectSegmentationMaskrcnn - model: /opt/openvino_toolkit/models/convert/public/mask_rcnn_inception_v2_coco_2018_01_28/OUT/frozen_inference_graph.xml + model: /opt/openvino_toolkit/models/public/mask_rcnn_inception_resnet_v2_atrous_coco/FP16/mask_rcnn_inception_resnet_v2_atrous_coco.xml engine: CPU #"HETERO:CPU,GPU,MYRIAD" label: to/be/set/xxx.labels batch: 1