diff --git a/csrc/fastdeploy/vision/common/processors/limit_short.h b/csrc/fastdeploy/vision/common/processors/limit_short.h index 51a26a1ba6..25eff6d717 100644 --- a/csrc/fastdeploy/vision/common/processors/limit_short.h +++ b/csrc/fastdeploy/vision/common/processors/limit_short.h @@ -34,6 +34,7 @@ class LimitShort : public Processor { static bool Run(Mat* mat, int max_short = -1, int min_short = -1, ProcLib lib = ProcLib::OPENCV_CPU); + int GetMaxShort() { return max_short_; } private: int max_short_; diff --git a/csrc/fastdeploy/vision/common/processors/resize_by_long.cc b/csrc/fastdeploy/vision/common/processors/resize_by_long.cc new file mode 100644 index 0000000000..7c2c90152f --- /dev/null +++ b/csrc/fastdeploy/vision/common/processors/resize_by_long.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "fastdeploy/vision/common/processors/resize_by_long.h" + +namespace fastdeploy { +namespace vision { + +bool ResizeByLong::CpuRun(Mat* mat) { + cv::Mat* im = mat->GetCpuMat(); + int origin_w = im->cols; + int origin_h = im->rows; + double scale = GenerateScale(origin_w, origin_h); + if (use_scale_) { + cv::resize(*im, *im, cv::Size(), scale, scale, interp_); + } else { + int width = static_cast(round(scale * im->cols)); + int height = static_cast(round(scale * im->rows)); + cv::resize(*im, *im, cv::Size(width, height), 0, 0, interp_); + } + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + return true; +} + +#ifdef ENABLE_OPENCV_CUDA +bool ResizeByLong::GpuRun(Mat* mat) { + cv::cuda::GpuMat* im = mat->GetGpuMat(); + int origin_w = im->cols; + int origin_h = im->rows; + double scale = GenerateScale(origin_w, origin_h); + im->convertTo(*im, CV_32FC(im->channels())); + if (use_scale_) { + cv::cuda::resize(*im, *im, cv::Size(), scale, scale, interp_); + } else { + int width = static_cast(round(scale * im->cols)); + int height = static_cast(round(scale * im->rows)); + cv::cuda::resize(*im, *im, cv::Size(width, height), 0, 0, interp_); + } + mat->SetWidth(im->cols); + mat->SetHeight(im->rows); + return true; +} +#endif + +double ResizeByLong::GenerateScale(const int origin_w, const int origin_h) { + int im_size_max = std::max(origin_w, origin_h); + int im_size_min = std::min(origin_w, origin_h); + double scale = 1.0f; + if (target_size_ == -1) { + if (im_size_max > max_size_) { + scale = static_cast(max_size_) / static_cast(im_size_max); + } + } else { + scale = + static_cast(target_size_) / static_cast(im_size_max); + } + return scale; +} + +bool ResizeByLong::Run(Mat* mat, int target_size, int interp, bool use_scale, + int max_size, ProcLib lib) { + auto r = ResizeByLong(target_size, interp, use_scale, max_size); + return r(mat, lib); +} +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/common/processors/resize_by_long.h b/csrc/fastdeploy/vision/common/processors/resize_by_long.h new file mode 100644 index 0000000000..5984bfca14 --- /dev/null +++ b/csrc/fastdeploy/vision/common/processors/resize_by_long.h @@ -0,0 +1,49 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "fastdeploy/vision/common/processors/base.h" + +namespace fastdeploy { +namespace vision { + +class ResizeByLong : public Processor { + public: + ResizeByLong(int target_size, int interp = 1, bool use_scale = true, + int max_size = -1) { + target_size_ = target_size; + max_size_ = max_size; + interp_ = interp; + use_scale_ = use_scale; + } + bool CpuRun(Mat* mat); +#ifdef ENABLE_OPENCV_CUDA + bool GpuRun(Mat* mat); +#endif + std::string Name() { return "ResizeByLong"; } + + static bool Run(Mat* mat, int target_size, int interp = 1, + bool use_scale = true, int max_size = -1, + ProcLib lib = ProcLib::OPENCV_CPU); + + private: + double GenerateScale(const int origin_w, const int origin_h); + int target_size_; + int max_size_; + int interp_; + bool use_scale_; +}; +} // namespace vision +} // namespace fastdeploy diff --git a/csrc/fastdeploy/vision/common/processors/transform.h b/csrc/fastdeploy/vision/common/processors/transform.h index 065006fbfb..cf720ceb7b 100644 --- a/csrc/fastdeploy/vision/common/processors/transform.h +++ b/csrc/fastdeploy/vision/common/processors/transform.h @@ -24,6 +24,7 @@ #include "fastdeploy/vision/common/processors/pad.h" #include "fastdeploy/vision/common/processors/pad_to_size.h" #include "fastdeploy/vision/common/processors/resize.h" +#include "fastdeploy/vision/common/processors/resize_by_long.h" #include "fastdeploy/vision/common/processors/resize_by_short.h" #include "fastdeploy/vision/common/processors/resize_to_int_mult.h" #include "fastdeploy/vision/common/processors/stride_pad.h" diff --git a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc index 63a9eff584..8c60b7ee00 100644 --- a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc +++ b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.cc @@ -26,8 +26,8 @@ PPMatting::PPMatting(const std::string& model_file, const RuntimeOption& custom_option, const Frontend& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; - valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + valid_cpu_backends = {Backend::ORT, Backend::PDINFER}; + valid_gpu_backends = {Backend::PDINFER, Backend::TRT}; runtime_option = custom_option; runtime_option.model_format = model_format; runtime_option.model_file = model_file; @@ -74,6 +74,11 @@ bool PPMatting::BuildPreprocessPipelineFromConfig() { if (op["min_short"]) { min_short = op["min_short"].as(); } + FDINFO << "Detected LimitShort processing step in yaml file, if the " + "model is exported from PaddleSeg, please make sure the " + "input of your model is fixed with a square shape, and " + "greater than or equal to " + << max_short << "." << std::endl; processors_.push_back( std::make_shared(max_short, min_short)); } else if (op["type"].as() == "ResizeToIntMult") { @@ -92,6 +97,19 @@ bool PPMatting::BuildPreprocessPipelineFromConfig() { std = op["std"].as>(); } processors_.push_back(std::make_shared(mean, std)); + } else if (op["type"].as() == "ResizeByLong") { + int target_size = op["target_size"].as(); + processors_.push_back(std::make_shared(target_size)); + } else if (op["type"].as() == "Pad") { + // size: (w, h) + auto size = op["size"].as>(); + std::vector value = {127.5, 127.5, 127.5}; + if (op["fill_value"]) { + auto value = op["fill_value"].as>(); + } + processors_.push_back(std::make_shared("float")); + processors_.push_back( + std::make_shared(size[1], size[0], value)); } } processors_.push_back(std::make_shared()); @@ -102,11 +120,30 @@ bool PPMatting::BuildPreprocessPipelineFromConfig() { bool PPMatting::Preprocess(Mat* mat, FDTensor* output, std::map>* im_info) { for (size_t i = 0; i < processors_.size(); ++i) { + if (processors_[i]->Name().compare("LimitShort") == 0) { + int input_h = static_cast(mat->Height()); + int input_w = static_cast(mat->Width()); + auto processor = dynamic_cast(processors_[i].get()); + int max_short = processor->GetMaxShort(); + if (runtime_option.backend != Backend::PDINFER) { + if (input_w != input_h || input_h < max_short || input_w < max_short) { + FDWARNING << "Detected LimitShort processing step in yaml file and " + "the size of input image is Unqualified, Fastdeploy " + "will resize the input image into (" + << max_short << "," << max_short << ")." << std::endl; + Resize::Run(mat, max_short, max_short); + } + } + } if (!(*(processors_[i].get()))(mat)) { FDERROR << "Failed to process image data in " << processors_[i]->Name() << "." << std::endl; return false; } + if (processors_[i]->Name().compare("ResizeByLong") == 0) { + (*im_info)["resize_by_long"] = {static_cast(mat->Height()), + static_cast(mat->Width())}; + } } // Record output shape of preprocessed image @@ -135,6 +172,7 @@ bool PPMatting::Postprocess( // 先获取alpha并resize (使用opencv) auto iter_ipt = im_info.find("input_shape"); auto iter_out = im_info.find("output_shape"); + auto resize_by_long = im_info.find("resize_by_long"); FDASSERT(iter_out != im_info.end() && iter_ipt != im_info.end(), "Cannot find input_shape or output_shape from im_info."); int out_h = iter_out->second[0]; @@ -145,7 +183,17 @@ bool PPMatting::Postprocess( // TODO: 需要修改成FDTensor或Mat的运算 现在依赖cv::Mat float* alpha_ptr = static_cast(alpha_tensor.Data()); cv::Mat alpha_zero_copy_ref(out_h, out_w, CV_32FC1, alpha_ptr); - Mat alpha_resized(alpha_zero_copy_ref); // ref-only, zero copy. + cv::Mat cropped_alpha; + if (resize_by_long != im_info.end()) { + int resize_h = resize_by_long->second[0]; + int resize_w = resize_by_long->second[1]; + alpha_zero_copy_ref(cv::Rect(0, 0, resize_w, resize_h)) + .copyTo(cropped_alpha); + } else { + cropped_alpha = alpha_zero_copy_ref; + } + Mat alpha_resized(cropped_alpha); // ref-only, zero copy. + if ((out_h != ipt_h) || (out_w != ipt_w)) { // already allocated a new continuous memory after resize. // cv::resize(alpha_resized, alpha_resized, cv::Size(ipt_w, ipt_h)); diff --git a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h index 148fe10e81..763d430770 100644 --- a/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h +++ b/csrc/fastdeploy/vision/matting/ppmatting/ppmatting.h @@ -27,7 +27,7 @@ class FASTDEPLOY_DECL PPMatting : public FastDeployModel { const RuntimeOption& custom_option = RuntimeOption(), const Frontend& model_format = Frontend::PADDLE); - std::string ModelName() const { return "PaddleMat"; } + std::string ModelName() const { return "PaddleMatting"; } virtual bool Predict(cv::Mat* im, MattingResult* result); diff --git a/examples/vision/matting/modnet/python/README.md b/examples/vision/matting/modnet/python/README.md index dc74487ac4..f22b779826 100644 --- a/examples/vision/matting/modnet/python/README.md +++ b/examples/vision/matting/modnet/python/README.md @@ -33,6 +33,7 @@ python infer.py --model modnet_photographic_portrait_matting.onnx --image mattin + ## MODNet Python接口 ```python diff --git a/examples/vision/matting/ppmatting/README.md b/examples/vision/matting/ppmatting/README.md index 0e12051150..d714322847 100644 --- a/examples/vision/matting/ppmatting/README.md +++ b/examples/vision/matting/ppmatting/README.md @@ -15,19 +15,18 @@ 在部署前，需要先将PPMatting导出成部署模型，导出步骤参考文档[导出模型](https://github.com/PaddlePaddle/PaddleSeg/tree/release/2.6/Matting) -注意：在导出模型时不要进行NMS的去除操作，正常导出即可。 ## 下载预训练模型 为了方便开发者的测试，下面提供了PPMatting导出的各系列模型，开发者可直接下载使用。 -其中精度指标来源于PPMatting中对各模型的介绍，详情各参考PPMatting中的说明。 +其中精度指标来源于PPMatting中对各模型的介绍(未提供精度数据)，详情各参考PPMatting中的说明。 | 模型 | 参数大小 | 精度 | 备注 | |:---------------------------------------------------------------- |:----- |:----- | :------ | -| [PPMatting](https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-512.tgz) | 87MB | - | -| [PPMatting](https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-1024.tgz) | 87MB | - | +| [PPMatting-512](https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-512.tgz) | 87MB | - | +| [PPMatting-1024](https://bj.bcebos.com/paddlehub/fastdeploy/PP-Matting-1024.tgz) | 87MB | - | diff --git a/examples/vision/matting/ppmatting/cpp/README.md b/examples/vision/matting/ppmatting/cpp/README.md index f189efe51f..726d8fcb36 100644 --- a/examples/vision/matting/ppmatting/cpp/README.md +++ b/examples/vision/matting/ppmatting/cpp/README.md @@ -27,7 +27,7 @@ wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_bgr.jpg # CPU推理 ./infer_demo PP-Matting-512 matting_input.jpg matting_bgr.jpg 0 -# GPU推理 (TODO: ORT-GPU 推理会报错) +# GPU推理 ./infer_demo PP-Matting-512 matting_input.jpg matting_bgr.jpg 1 # GPU上TensorRT推理 ./infer_demo PP-Matting-512 matting_input.jpg matting_bgr.jpg 2 diff --git a/examples/vision/matting/ppmatting/cpp/infer.cc b/examples/vision/matting/ppmatting/cpp/infer.cc index bb8c6eac28..f47b484e5e 100644 --- a/examples/vision/matting/ppmatting/cpp/infer.cc +++ b/examples/vision/matting/ppmatting/cpp/infer.cc @@ -25,8 +25,9 @@ void CpuInfer(const std::string& model_dir, const std::string& image_file, auto model_file = model_dir + sep + "model.pdmodel"; auto params_file = model_dir + sep + "model.pdiparams"; auto config_file = model_dir + sep + "deploy.yaml"; + auto option = fastdeploy::RuntimeOption(); auto model = fastdeploy::vision::matting::PPMatting(model_file, params_file, - config_file); + config_file, option); if (!model.Initialized()) { std::cerr << "Failed to initialize." << std::endl; return; @@ -58,6 +59,7 @@ void GpuInfer(const std::string& model_dir, const std::string& image_file, auto option = fastdeploy::RuntimeOption(); option.UseGpu(); + option.UsePaddleBackend(); auto model = fastdeploy::vision::matting::PPMatting(model_file, params_file, config_file, option); if (!model.Initialized()) { diff --git a/examples/vision/matting/ppmatting/python/README.md b/examples/vision/matting/ppmatting/python/README.md index 573a799483..427a3dcd47 100644 --- a/examples/vision/matting/ppmatting/python/README.md +++ b/examples/vision/matting/ppmatting/python/README.md @@ -19,7 +19,7 @@ wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_input.jpg wget https://bj.bcebos.com/paddlehub/fastdeploy/matting_bgr.jpg # CPU推理 python infer.py --model PP-Matting-512 --image matting_input.jpg --bg matting_bgr.jpg --device cpu -# GPU推理 (TODO: ORT-GPU 推理会报错) +# GPU推理 python infer.py --model PP-Matting-512 --image matting_input.jpg --bg matting_bgr.jpg --device gpu # GPU上使用TensorRT推理 （注意：TensorRT推理第一次运行，有序列化模型的操作，有一定耗时，需要耐心等待） python infer.py --model PP-Matting-512 --image matting_input.jpg --bg matting_bgr.jpg --device gpu --use_trt True diff --git a/examples/vision/matting/ppmatting/python/infer.py b/examples/vision/matting/ppmatting/python/infer.py index 4f5e7ffc4e..b60fcbc820 100644 --- a/examples/vision/matting/ppmatting/python/infer.py +++ b/examples/vision/matting/ppmatting/python/infer.py @@ -32,13 +32,14 @@ def parse_arguments(): def build_option(args): option = fd.RuntimeOption() - if args.device.lower() == "gpu": option.use_gpu() + option.use_paddle_backend() if args.use_trt: option.use_trt_backend() option.set_trt_input_shape("img", [1, 3, 512, 512]) + return option diff --git a/external/paddle_inference.cmake b/external/paddle_inference.cmake index 279b22e3bb..3f7183effc 100644 --- a/external/paddle_inference.cmake +++ b/external/paddle_inference.cmake @@ -46,6 +46,7 @@ else() set(OMP_LIB "${PADDLEINFERENCE_INSTALL_DIR}/third_party/install/mklml/lib/libiomp5.so") endif(WIN32) + set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/") set(PADDLEINFERENCE_VERSION "2.4-dev") if(WIN32)