chore: review 新增的推理后端coreML和paddle-lite

nndeploy · Oct 6, 2023 · 075dd5b · 075dd5b
1 parent 00c4c58
commit 075dd5b
Show file tree

Hide file tree

Showing 12 changed files with 191 additions and 122 deletions.
diff --git a/README.md b/README.md
@@ -26,17 +26,19 @@
 
 ### 支持多平台和多推理框架
 
-- 支持多种推理框架：对多个业界知名推理框架的全面支持，包括 `TensorRT`、`OpenVINO`、`ONNXRuntime`、`MNN`、`TNN`、`ncnn` 等。未来，我们将继续扩展支持，包括 `TFLite`、`paddle-lite`、`coreML`、`TVM`、`AITemplate`、`RKNN`等
-- 支持多种不同操作系统，包括 `Android`、`Linux`、`Windows`，正在适配 `macOS`、`IOS`。致力于在各种操作系统上无缝运行您的深度学习模型
-
-|                      OS/Inference                       | Linux | Windows | Android | MacOS |  IOS  |                 开发人员                  | 备注  |
-| :-----------------------------------------------------: | :---: | :-----: | :-----: | :---: | :---: | :---------------------------------------: | :---: |
-|     [TensorRT](https://github.com/NVIDIA/TensorRT)      |  yes  |   no    |   no    |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
-| [OpenVINO](https://github.com/openvinotoolkit/openvino) |  yes  |   yes   |   no    |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
-| [ONNXRuntime](https://github.com/microsoft/onnxruntime) |  yes  |   yes   |   no    |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
-|          [MNN](https://github.com/alibaba/MNN)          |  yes  |   yes   |   yes   |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
-|          [TNN](https://github.com/Tencent/TNN)          |  yes  |   yes   |   yes   |  no   |  no   | [02200059Z](https://github.com/02200059Z) |       |
-|        [ncnn](https://github.com/Tencent/ncnn/)         |  no   |   no    |   yes   |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
+- 支持多种推理框架：对多个业界知名推理框架的全面支持，包括 `TensorRT`、`OpenVINO`、`ONNXRuntime`、`MNN`、`TNN`、`ncnn` 、`coreML`、`paddle-lite`。未来，我们将继续扩展支持，包括 `TFLite`、`TVM`、`OpenPPL`、`Tengine`、`AITemplate`、`RKNN`、`sophgo`等
+- 支持多种不同操作系统，包括 `Android`、`Linux`、`Windows`、`macOS`，正在适配 `IOS`。致力于在各种操作系统上无缝运行您的深度学习模型
+
+|                        OS/Inference                        |  Linux   | Windows | Android | MacOS |  IOS  |                 开发人员                  | 备注  |
+| :--------------------------------------------------------: | :------: | :-----: | :-----: | :---: | :---: | :---------------------------------------: | :---: |
+|       [TensorRT](https://github.com/NVIDIA/TensorRT)       |   yes    |   no    |   no    |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
+|  [OpenVINO](https://github.com/openvinotoolkit/openvino)   |   yes    |   yes   |   no    |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
+|  [ONNXRuntime](https://github.com/microsoft/onnxruntime)   |   yes    |   yes   |   no    |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
+|           [MNN](https://github.com/alibaba/MNN)            |   yes    |   yes   |   yes   |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
+|           [TNN](https://github.com/Tencent/TNN)            |   yes    |   yes   |   yes   |  no   |  no   | [02200059Z](https://github.com/02200059Z) |       |
+|          [ncnn](https://github.com/Tencent/ncnn)           |    no    |   no    |   yes   |  no   |  no   | [Always](https://github.com/Alwaysssssss) |       |
+|       [coreML](https://github.com/apple/coremltools)       |    no    |   no    |   no    |  yes  |  no   |  [JoDio-zd](https://github.com/JoDio-zd)  |       |
+| [paddle-lite](https://github.com/PaddlePaddle/Paddle-Lite) | nocoreml |   no    |   no    |  no   |  no   | [qixuxiang](https://github.com/qixuxiang) |       |
 
 ### 直接可用的算法
 
@@ -188,7 +190,7 @@
 
 ## TODO
 
-- 接入更多的推理框架，包括 `TFLite`、`paddle-lite`、`coreML`、`TVM`、`AITemplate`、`RKNN`、算能等等推理软件栈
+- 接入更多的推理框架，包括 `TFLite`、`TVM`、`OpenPPL`、`Tengine`、`AITemplate`、`RKNN`、`sophgo`等等推理软件栈
 - 部署更多的算法，包括 `Stable Diffusion`、`DETR`、`SAM`等等热门开源模型
 
 ## 快速开始

diff --git a/include/nndeploy/inference/coreml/coreml_inference.h b/include/nndeploy/inference/coreml/coreml_inference.h
@@ -31,16 +31,16 @@ class CoremlInference : public Inference {
   virtual base::Status init();
   virtual base::Status deinit();
 
-  virtual base::Status reshape(base::ShapeMap &shape_map);
+  virtual base::Status reshape(base::ShapeMap &shape_map);  // review: 不支持动态shape吗？
 
-  virtual int64_t getMemorySize();
+  virtual int64_t getMemorySize();  // review： 返回-1会不会更好呀
 
-  virtual float getGFLOPs();
+  virtual float getGFLOPs();  // review： 返回-1会不会更好呀
 
   virtual device::TensorDesc getInputTensorAlignDesc(const std::string &name);
   virtual device::TensorDesc getOutputTensorAlignDesc(const std::string &name);
 
-  virtual base::Status run();
+  virtual base::Status run();  // review：怎么看起来只能支持 int8的输入 tensor呀
 
  private:
   base::Status allocateInputOutputTensor();

diff --git a/include/nndeploy/inference/coreml/coreml_inference_param.h b/include/nndeploy/inference/coreml/coreml_inference_param.h
@@ -26,8 +26,10 @@ class CoremlInferenceParam : public InferenceParam {
 
   /// @brief A Boolean value that determines whether to allow low-precision
   /// accumulation on a GPU.
-  bool low_precision_acceleration_ = false;
-  enum inferenceUnits {
+  bool low_precision_acceleration_ =
+      false;  // reivew： 可否使用InferenceParam::precision_type_
+  enum inferenceUnits {  // reivew： 可否使用InferenceParam::device_type_,
+                         // 参照openvino的做法，继续沿用该做法的法，需要inferenceUnits->InferenceUnits
     ALL_UNITS = 0,
     CPU_ONLY = 1,
     CPU_AND_GPU = 2,

diff --git a/include/nndeploy/inference/inference_param.h b/include/nndeploy/inference/inference_param.h
@@ -104,8 +104,8 @@ class TypeInferenceParamCreator : public InferenceParamCreator {
  * @return std::map<base::InferenceType,
  * std::shared_ptr<InferenceParamCreator>>&
  */
-std::map<base::InferenceType, std::shared_ptr<InferenceParamCreator>> &
-getGlobalInferenceParamCreatorMap();
+std::map<base::InferenceType, std::shared_ptr<InferenceParamCreator>>
+    &getGlobalInferenceParamCreatorMap();
 
 /**
  * @brief TypeInferenceParamRegister is the template class of all inference

diff --git a/include/nndeploy/inference/paddlelite/paddlelite_convert.h b/include/nndeploy/inference/paddlelite/paddlelite_convert.h
@@ -11,25 +11,29 @@
 #include "nndeploy/inference/inference_param.h"
 #include "nndeploy/inference/paddlelite/paddlelite_include.h"
 
-
 namespace nndeploy {
 namespace inference {
 
 class PaddleLiteConvert {
  public:
-  static base::DataType convertToDataType(const paddle::lite_api::PrecisionType &src);
-  static paddle::lite_api::PrecisionType convertFromDataType(const base::DataType &src);
+  static base::DataType convertToDataType(
+      const paddle::lite_api::PrecisionType &src);
+  static paddle::lite_api::PrecisionType convertFromDataType(
+      const base::DataType &src);
 
-  static paddle::lite::TargetType convertFromDeviceType(const base::DeviceType &src);
-  static base::DeviceType convertToDeviceType(const paddle::lite::TargetType &src);
-
+  static paddle::lite::TargetType convertFromDeviceType(
+      const base::DeviceType &src);
+  static base::DeviceType convertToDeviceType(
+      const paddle::lite::TargetType &src);
 
   static base::DataFormat convertToDataFormat(
       const paddle::lite_api::DataLayoutType &src);
 
   static base::IntVector convertToShape(const paddle::lite::DDim &src);
   static paddle::lite::DDim convertFromShape(const base::IntVector &src);
 
+  static base::Status convertFromInferenceParam(
+      PaddleLiteInferenceParam *src, paddle::lite_api::CxxConfig &dst);
 };
 
 }  // namespace inference

diff --git a/include/nndeploy/inference/paddlelite/paddlelite_include.h b/include/nndeploy/inference/paddlelite/paddlelite_include.h
@@ -2,12 +2,11 @@
 #ifndef _NNDEPLOY_INFERENCE_PADDLELITE_PADDLELITE_INCLUDE_H_
 #define _NNDEPLOY_INFERENCE_PADDLELITE_PADDLELITE_INCLUDE_H_
 
-#include <lite/api/paddle_api.h>
 #include <lite/api/cxx_api.h>
-#include <lite/core/types.h>
-#include <lite/core/tensor.h>
+#include <lite/api/paddle_api.h>
 #include <lite/core/context.h>
 #include <lite/core/program.h>
-
+#include <lite/core/tensor.h>
+#include <lite/core/types.h>
 
 #endif
diff --git a/include/nndeploy/inference/paddlelite/paddlelite_inference.h b/include/nndeploy/inference/paddlelite/paddlelite_inference.h
@@ -31,15 +31,17 @@ class PaddleLiteInference : public Inference {
 
   virtual base::Status run();
 
-private:
-    paddle::lite_api::MobileConfig config_;
-    std::shared_ptr<paddle::lite_api::PaddlePredictor> predictor_;
-    size_t power_mode_ = 0;
-
+ private:
+  base::Status allocateInputOutputTensor();
+  base::Status deallocateInputOutputTensor();
+
+ private:
+  paddle::lite_api::CxxConfig config_;
+  std::shared_ptr<paddle::lite_api::PaddlePredictor> predictor_;
+  std::map<std::string, int> io_name_index_;
 };
 
 }  // namespace inference
 }  // namespace nndeploy
 
-
 #endif
diff --git a/source/nndeploy/inference/coreml/coreml_convert.mm b/source/nndeploy/inference/coreml/coreml_convert.mm
@@ -4,6 +4,10 @@
 namespace nndeploy {
 namespace inference {
 
+/**
+ * @brief Convert from base::DataType to MLFeatureType
+ * @review：只有这一种类型吗？
+ */
 base::DataType CoremlConvert::convertToDataType(const OSType &src) {
   base::DataType dst;
   switch (src) {
@@ -19,19 +23,22 @@
   return dst;
 }
 
+/**
+ * @review：
+ */
 NSObject *CoremlConvert::convertFromDeviceType(const base::DeviceType &src) {
   NSObject *type = nil;
   switch (src.code_) {
     case base::kDeviceTypeCodeCpu:
     case base::kDeviceTypeCodeX86:
     case base::kDeviceTypeCodeArm:
-    case base::kDeviceTypeCodeOpenCL:
+    case base::kDeviceTypeCodeOpenCL:  // review： 这个应该属于GPU吧？或者属于default
       type = reinterpret_cast<NSObject *>(new MLCPUComputeDevice());
       break;
-    case base::kDeviceTypeCodeOpenGL:
+    case base::kDeviceTypeCodeOpenGL:  // review： 属于default会不会更好呀
     case base::kDeviceTypeCodeMetal:
-    case base::kDeviceTypeCodeCuda:
-    case base::kDeviceTypeCodeVulkan:
+    case base::kDeviceTypeCodeCuda:    // review： 属于default会不会更好呀
+    case base::kDeviceTypeCodeVulkan:  // review： 属于default会不会更好呀
       type = reinterpret_cast<NSObject *>(new MLGPUComputeDevice());
       break;
     case base::kDeviceTypeCodeNpu:
@@ -43,8 +50,8 @@
   return type;
 }
 
-base::Status CoremlConvert::convertFromInferenceParam(
-    CoremlInferenceParam *src, MLModelConfiguration *dst) {
+base::Status CoremlConvert::convertFromInferenceParam(CoremlInferenceParam *src,
+                                                      MLModelConfiguration *dst) {
   dst.allowLowPrecisionAccumulationOnGPU = src->low_precision_acceleration_;
   switch (src->inference_units_) {
     case CoremlInferenceParam::inferenceUnits::ALL_UNITS:
@@ -67,23 +74,21 @@
 }
 
 device::Tensor *CoremlConvert::convertToTensor(MLFeatureDescription *src, NSString *name,
-                                device::Device *device) {
+                                               device::Device *device) {
   MLFeatureType tensor_type = [src type];
   device::Tensor *dst = nullptr;
   device::TensorDesc desc;
   switch (tensor_type) {
-    case MLFeatureTypeImage:
-    {
+    case MLFeatureTypeImage: {
       MLImageConstraint *image_attr = [src imageConstraint];
       base::DataType data_type = CoremlConvert::convertToDataType([image_attr pixelFormatType]);
       base::DataFormat format = base::kDataFormatNHWC;
-      base::IntVector shape = {1, int([image_attr pixelsHigh]), int ([image_attr pixelsWide]), 3};
+      base::IntVector shape = {1, int([image_attr pixelsHigh]), int([image_attr pixelsWide]), 3};
       base::SizeVector stride = base::SizeVector();
       desc = device::TensorDesc(data_type, format, shape, stride);
       break;
     }
-    case MLFeatureTypeDouble:
-    {
+    case MLFeatureTypeDouble: {
       base::DataType data_type = base::DataType();
       base::DataFormat format = base::kDataFormatN;
       base::IntVector shape = {1};

diff --git a/source/nndeploy/inference/coreml/coreml_inference.mm b/source/nndeploy/inference/coreml/coreml_inference.mm
@@ -4,8 +4,8 @@
 namespace nndeploy {
 namespace inference {
 
-TypeInferenceRegister<TypeInferenceCreator<CoremlInference>>
-    g_coreml_inference_register(base::kInferenceTypeCoreML);
+TypeInferenceRegister<TypeInferenceCreator<CoremlInference>> g_coreml_inference_register(
+    base::kInferenceTypeCoreML);
 
 CoremlInference::CoremlInference(base::InferenceType type) : Inference(type) {}
 CoremlInference::~CoremlInference() {}
@@ -26,13 +26,10 @@
   CoremlConvert::convertFromInferenceParam(coreml_inference_param, config_);
 
   if (inference_param_->is_path_) {
-    NSURL *model_path = [NSURL
-        fileURLWithPath:[NSString
-                            stringWithCString:inference_param_->model_value_[0]
-                                                  .c_str() encoding:NSASCIIStringEncoding]];
-    mlmodel_ = [MLModel modelWithContentsOfURL:model_path
-                                 configuration:config_
-                                         error:&err_];
+    NSURL *model_path =
+        [NSURL fileURLWithPath:[NSString stringWithCString:inference_param_->model_value_[0].c_str()
+                                                  encoding:NSASCIIStringEncoding]];
+    mlmodel_ = [MLModel modelWithContentsOfURL:model_path configuration:config_ error:&err_];
     CHECK_ERR(err_);
   } else {
     NNDEPLOY_LOGI("You will load model from memory\n");
@@ -61,16 +58,13 @@
   return status;
 }
 
-base::Status CoremlInference::reshape(base::ShapeMap &shape_map) {
-  return base::kStatusCodeOk;
-}
+base::Status CoremlInference::reshape(base::ShapeMap &shape_map) { return base::kStatusCodeOk; }
 
 int64_t CoremlInference::getMemorySize() { return 0; }
 
 float CoremlInference::getGFLOPs() { return 1000.f; }
 
-device::TensorDesc CoremlInference::getInputTensorAlignDesc(
-    const std::string &name) {
+device::TensorDesc CoremlInference::getInputTensorAlignDesc(const std::string &name) {
   if (input_tensors_.count(name) > 0) {
     device::TensorDesc desc = input_tensors_[name]->getDesc();
     if (desc.shape_.size() == 5) {
@@ -84,8 +78,7 @@
         desc.data_format_ = base::kDataFormatNCHW;
       }
     } else if (desc.shape_.size() == 3) {
-      if (desc.data_format_ != base::kDataFormatNHW &&
-          desc.data_format_ != base::kDataFormatNWC &&
+      if (desc.data_format_ != base::kDataFormatNHW && desc.data_format_ != base::kDataFormatNWC &&
           desc.data_format_ != base::kDataFormatNCW) {
         desc.data_format_ = base::kDataFormatNHW;
       }
@@ -106,8 +99,7 @@
   }
 }
 
-device::TensorDesc CoremlInference::getOutputTensorAlignDesc(
-    const std::string &name) {
+device::TensorDesc CoremlInference::getOutputTensorAlignDesc(const std::string &name) {
   if (output_tensors_.count(name) > 0) {
     device::TensorDesc desc = output_tensors_[name]->getDesc();
     if (desc.shape_.size() == 5) {
@@ -121,8 +113,7 @@
         desc.data_format_ = base::kDataFormatNCHW;
       }
     } else if (desc.shape_.size() == 3) {
-      if (desc.data_format_ != base::kDataFormatNHW &&
-          desc.data_format_ != base::kDataFormatNWC &&
+      if (desc.data_format_ != base::kDataFormatNHW && desc.data_format_ != base::kDataFormatNWC &&
           desc.data_format_ != base::kDataFormatNCW) {
         desc.data_format_ = base::kDataFormatNHW;
       }
@@ -153,23 +144,25 @@
     int height = iter.second->getHeight();
     int stride = width;
     OSType pixelFormat = kCVPixelFormatType_OneComponent8;
-    CVReturn status = CVPixelBufferCreateWithBytes(
-        kCFAllocatorDefault, width, height, pixelFormat, iter.second->getPtr(),
-        stride, NULL, NULL, NULL, &photodata);
+    CVReturn status =
+        CVPixelBufferCreateWithBytes(kCFAllocatorDefault, width, height, pixelFormat,
+                                     iter.second->getPtr(), stride, NULL, NULL, NULL, &photodata);
     if (status != 0) {
       NNDEPLOY_LOGE("Tensor create failed");
     }
-    MLFeatureValue* input_data = [MLFeatureValue featureValueWithPixelBuffer:photodata];
+    MLFeatureValue *input_data = [MLFeatureValue featureValueWithPixelBuffer:photodata];
     [dict_ setObject:[NSString stringWithCString:iter.first.c_str() encoding:NSASCIIStringEncoding]
-             forKey:input_data];
+              forKey:input_data];
   }
   MLDictionaryFeatureProvider *provider =
-      [[MLDictionaryFeatureProvider alloc] initWithDictionary:dict_
-                                                        error:&err_];
-  NSDictionary<NSString *, MLFeatureValue *>* res = [[mlmodel_ predictionFromFeatures:provider error:&err_] dictionary];
+      [[MLDictionaryFeatureProvider alloc] initWithDictionary:dict_ error:&err_];
+  NSDictionary<NSString *, MLFeatureValue *> *res =
+      [[mlmodel_ predictionFromFeatures:provider error:&err_] dictionary];
   for (auto iter : external_output_tensors_) {
-    MLFeatureValue *value = res[[NSString stringWithCString:iter.first.c_str() encoding:NSASCIIStringEncoding]];
-    [&](void *&& data) -> void *& {return data;}(iter.second->getPtr()) = CVPixelBufferGetBaseAddress([value imageBufferValue]);
+    MLFeatureValue *value =
+        res[[NSString stringWithCString:iter.first.c_str() encoding:NSASCIIStringEncoding]];
+    [&](void *&&data) -> void *& { return data; }(iter.second->getPtr()) =
+                             CVPixelBufferGetBaseAddress([value imageBufferValue]);
   }
   return base::kStatusCodeOk;
 }
@@ -180,14 +173,17 @@
     device = device::getDevice(inference_param_->device_type_);
   }
   MLModelDescription *model_description = [mlmodel_ modelDescription];
-  NSDictionary<NSString *, MLFeatureDescription *> *model_input_feature = [model_description inputDescriptionsByName];
-  for (NSString * iter in model_input_feature) {
+  NSDictionary<NSString *, MLFeatureDescription *> *model_input_feature =
+      [model_description inputDescriptionsByName];
+  for (NSString *iter in model_input_feature) {
     device::Tensor *input_tensor =
-            CoremlConvert::convertToTensor(model_input_feature[iter], iter, device);
-    input_tensors_.insert({std::string([iter cStringUsingEncoding:NSASCIIStringEncoding]), input_tensor});
+        CoremlConvert::convertToTensor(model_input_feature[iter], iter, device);
+    input_tensors_.insert(
+        {std::string([iter cStringUsingEncoding:NSASCIIStringEncoding]), input_tensor});
   }
-  NSDictionary<NSString *, MLFeatureDescription *> *model_output_feature = [model_description outputDescriptionsByName];
-  for (NSString * iter in model_output_feature) {
+  NSDictionary<NSString *, MLFeatureDescription *> *model_output_feature =
+      [model_description outputDescriptionsByName];
+  for (NSString *iter in model_output_feature) {
     device::Tensor *dst = CoremlConvert::convertToTensor(model_input_feature[iter], iter, device);
     output_tensors_.insert({std::string([iter cStringUsingEncoding:NSASCIIStringEncoding]), dst});
   }