[Model] Integrate FlyCV image processing library (#375)

* Add falcon cmake * Update CMakeLists.txt * Add implementation for more preprocessors * Add more implementation * Add more implementations by flycv * Fix some name * fix some bug * ready to test * add fuse * fix fuse problem * add debug code * add debug * [Android] integrate flycv to Android (#459) * [Android] integrate flycv into Andorid * Falconcv (#27) * [Bug Fix ]Update ppseg visualize python api && demo (#454) Update ppseg visualize python api && demo * [Model] update pptracking letterboxresize and add some comment (#438) * add override mark * delete some * recovery * recovery * add tracking * add tracking py_bind and example * add pptracking * add pptracking * iomanip head file * add opencv_video lib * add python libs package Signed-off-by: ChaoII <849453582@qq.com> * complete comments Signed-off-by: ChaoII <849453582@qq.com> * add jdeTracker_ member variable Signed-off-by: ChaoII <849453582@qq.com> * add 'FASTDEPLOY_DECL' macro Signed-off-by: ChaoII <849453582@qq.com> * remove kwargs params Signed-off-by: ChaoII <849453582@qq.com> * [Doc]update pptracking docs * delete 'ENABLE_PADDLE_FRONTEND' switch * add pptracking unit test * update pptracking unit test Signed-off-by: ChaoII <849453582@qq.com> * modify test video file path and remove trt test * update unit test model url * remove 'FASTDEPLOY_DECL' macro Signed-off-by: ChaoII <849453582@qq.com> * fix build python packages about pptracking on win32 Signed-off-by: ChaoII <849453582@qq.com> * update comment Signed-off-by: ChaoII <849453582@qq.com> * add pptracking model explain Signed-off-by: ChaoII <849453582@qq.com> Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: Jason <jiangjiajun@baidu.com> Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: ChaoII <849453582@qq.com> Co-authored-by: Jason <jiangjiajun@baidu.com> * [cmake] optimize examples CMakeLists.txt Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: ChaoII <849453582@qq.com> Co-authored-by: Jason <jiangjiajun@baidu.com> * remove debug log * remove function * remove function Signed-off-by: ChaoII <849453582@qq.com> Co-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com> Co-authored-by: huangjianhui <852142024@qq.com> Co-authored-by: ChaoII <849453582@qq.com>
PaddlePaddle · Nov 1, 2022 · bb00e07 · bb00e07
1 parent c76992c
commit bb00e07
Show file tree

Hide file tree

Showing 68 changed files with 2,571 additions and 106 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -60,8 +60,8 @@ option(ENABLE_POROS_BACKEND "Whether to enable poros backend." OFF)
 option(ENABLE_OPENVINO_BACKEND "Whether to enable openvino backend." OFF)
 option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF)
 option(ENABLE_VISION "Whether to enable vision models usage." OFF)
-option(ENABLE_VISION_VISUALIZE "Whether to enable visualize vision model result toolbox." ON)
 option(ENABLE_TEXT "Whether to enable text models usage." OFF)
+option(ENABLE_FLYCV "Whether to enable flycv to boost image preprocess." OFF)
 option(WITH_TESTING "Whether to compile with unittest." OFF)
 ############################# Options for Android cross compiling #########################
 option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF)
@@ -136,16 +136,6 @@ set(HEAD_DIR "${PROJECT_SOURCE_DIR}/${CSRCS_DIR_NAME}")
 include_directories(${HEAD_DIR})
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-if(BUILD_EXAMPLES AND EXISTS ${PROJECT_SOURCE_DIR}/examples)
-  if(ENABLE_VISION)
-    # ENABLE_VISION_VISUALIZE must be ON if enable vision examples.
-    message(STATUS "Found BUILD_EXAMPLES and ENABLE_VISION ON, so, force ENABLE_VISION_VISUALIZE ON")
-    set(ENABLE_VISION_VISUALIZE ON CACHE BOOL "force to enable visualize vision model result toolbox" FORCE)
-  else()
-    message(WARNING "BUILD_EXAMPLES is ON, but found ENABLE_VISION OFF, will skip vision examples.")
-  endif()
-endif()
-
 if(ANDROID OR IOS)
   if(ENABLE_ORT_BACKEND)
     message(FATAL_ERROR "Not support ONNXRuntime backend for Andorid/IOS now. Please set ENABLE_ORT_BACKEND=OFF.")
@@ -382,6 +372,8 @@ endif()
 
 if(ENABLE_VISION)
   add_definitions(-DENABLE_VISION)
+#  set(ENABLE_VISION_VISUALIZE ON)
+  add_definitions(-DENABLE_VISION_VISUALIZE)
   if(ENABLE_OPENCV_CUDA)
     if(NOT WITH_GPU)
       message(FATAL_ERROR "ENABLE_OPENCV_CUDA is available on Linux and WITH_GPU=ON, but now WITH_GPU=OFF.")
@@ -402,13 +394,10 @@ if(ENABLE_VISION)
   include_directories(${PROJECT_SOURCE_DIR}/third_party/yaml-cpp/include)
   include(${PROJECT_SOURCE_DIR}/cmake/opencv.cmake)
 
-  if(ENABLE_VISION_VISUALIZE)
-    add_definitions(-DENABLE_VISION_VISUALIZE)
-  endif()
-else()
-  if(ENABLE_VISION_VISUALIZE)
-    message("While ENABLE_VISION=OFF, will force ENABLE_VISION_VISUALIZE=OFF.")
-    set(ENABLE_VISION_VISUALIZE OFF)
+  if(ENABLE_FLYCV)
+    add_definitions(-DENABLE_FLYCV)
+    include(${PROJECT_SOURCE_DIR}/cmake/flycv.cmake)
+    list(APPEND DEPEND_LIBS external_flycv)
   endif()
 endif()
 

diff --git a/FastDeploy.cmake.in b/FastDeploy.cmake.in
@@ -10,6 +10,7 @@ set(POROS_VERSION @POROS_VERSION@)
 set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND@)
 set(ENABLE_PADDLE_FRONTEND @ENABLE_PADDLE_FRONTEND@)
 set(ENABLE_VISION @ENABLE_VISION@)
+set(ENABLE_FLYCV @ENABLE_FLYCV@)
 set(ENABLE_TEXT @ENABLE_TEXT@)
 set(BUILD_ON_JETSON @BUILD_ON_JETSON@)
 set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@)
@@ -185,6 +186,20 @@ if(ENABLE_VISION)
     list(APPEND FASTDEPLOY_INCS ${OpenCV_INCLUDE_DIRS})
     list(APPEND FASTDEPLOY_LIBS ${OpenCV_LIBS})
   endif()
+
+  if(ENABLE_FLYCV)
+    include_directories(${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/include)
+    set(FLYCV_LIB_DIR ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/flycv/lib)
+    if(ANDROID)
+      add_library(flycv_shared STATIC IMPORTED GLOBAL)
+      set_property(TARGET flycv_shared PROPERTY IMPORTED_LOCATION ${FLYCV_LIB_DIR}/${ANDROID_ABI}/libflycv_shared.so)
+      list(APPEND FASTDEPLOY_LIBS flycv_shared)
+    else()
+      find_library(FLYCV_LIB flycv_shared ${FLYCV_LIB_DIR} NO_DEFAULT_PATH)
+      list(APPEND FASTDEPLOY_LIBS ${FLYCV_LIB})
+    endif()
+  endif()
+
 endif()
 
 if (ENABLE_TEXT)

diff --git a/cmake/flycv.cmake b/cmake/flycv.cmake
@@ -0,0 +1,130 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include(ExternalProject)
+
+set(FLYCV_PROJECT "extern_flycv")
+set(FLYCV_PREFIX_DIR ${THIRD_PARTY_PATH}/flycv)
+set(FLYCV_SOURCE_DIR
+    ${THIRD_PARTY_PATH}/flycv/src/${FLYCV_PROJECT})
+set(FLYCV_INSTALL_DIR ${THIRD_PARTY_PATH}/install/flycv)
+set(FLYCV_INC_DIR
+    "${FLYCV_INSTALL_DIR}/include"
+    CACHE PATH "flycv include directory." FORCE)
+if(ANDROID)    
+  set(FLYCV_LIB_DIR
+      "${FLYCV_INSTALL_DIR}/lib/${ANDROID_ABI}"
+      CACHE PATH "flycv lib directory." FORCE)
+else()
+  set(FLYCV_LIB_DIR
+        "${FLYCV_INSTALL_DIR}/lib/"
+        CACHE PATH "flycv lib directory." FORCE)
+endif()    
+set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}"
+                      "${FLYCV_LIB_DIR}")
+
+include_directories(${FLYCV_INC_DIR})
+
+# ABI check
+if(ANDROID)
+  if((NOT ANDROID_ABI MATCHES "armeabi-v7a") AND (NOT ANDROID_ABI MATCHES "arm64-v8a"))
+    message(FATAL_ERROR "FastDeploy with FlyCV only support armeabi-v7a, arm64-v8a now.")
+  endif()
+  if(NOT ANDROID_TOOLCHAIN MATCHES "clang")
+    message(FATAL_ERROR "Currently, only support clang toolchain while cross compiling FastDeploy for Android with FlyCV, but found ${ANDROID_TOOLCHAIN}.")
+  endif()  
+endif()
+
+if(WIN32)
+  set(FLYCV_COMPILE_LIB
+      "${FLYCV_INSTALL_DIR}/lib/flycv.lib"
+      CACHE FILEPATH "flycv compile library." FORCE)
+elseif(APPLE)
+  set(FLYCV_COMPILE_LIB
+      "${FLYCV_INSTALL_DIR}/lib/libflycv.dylib"
+      CACHE FILEPATH "flycv compile library." FORCE)      
+elseif(ANDROID)
+  set(FLYCV_COMPILE_LIB
+  "${FLYCV_INSTALL_DIR}/lib/${ANDROID_ABI}/libflycv_shared.so"
+  CACHE FILEPATH "flycv compile library." FORCE)   
+else()
+  set(FLYCV_COMPILE_LIB
+      "${FLYCV_INSTALL_DIR}/lib/libflycv_shared.so"
+      CACHE FILEPATH "flycv compile library." FORCE)
+endif(WIN32)
+
+set(FLYCV_URL_BASE "https://bj.bcebos.com/fastdeploy/third_libs/")
+set(FLYCV_VERSION "1.0.1025")
+if(WIN32)
+  message(FATAL_ERROR "FlyCV is not supported on Windows now.")
+  set(FLYCV_FILE "flycv-win-x64-${FLYCV_VERSION}.zip")
+elseif(APPLE)
+  message(FATAL_ERROR "FlyCV is not supported on Mac OSX now.")
+  if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
+    set(FLYCV_FILE "flycv-osx-arm64-${FLYCV_VERSION}.tgz")
+  else()
+    set(FLYCV_FILE "flycv-osx-x86_64-${FLYCV_VERSION}.tgz")
+  endif()
+elseif(ANDROID)
+  set(FLYCV_FILE "flycv-android-${FLYCV_VERSION}.tgz")  
+else()
+  if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
+    set(FLYCV_FILE "flycv-linux-aarch64-${FLYCV_VERSION}.tgz")
+  else()
+    message(FATAL_ERROR "FlyCV is not supported on Linux x64 now.")
+    set(FLYCV_FILE "flycv-linux-x64-${FLYCV_VERSION}.tgz")
+  endif()
+endif()
+set(FLYCV_URL "${FLYCV_URL_BASE}${FLYCV_FILE}")
+
+if(ANDROID)
+  ExternalProject_Add(
+    ${FLYCV_PROJECT}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    URL ${FLYCV_URL}
+    PREFIX ${FLYCV_PREFIX_DIR}
+    DOWNLOAD_NO_PROGRESS 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND
+      ${CMAKE_COMMAND} -E remove_directory ${FLYCV_INSTALL_DIR} &&
+      ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR} &&
+      ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR}/lib &&
+      ${CMAKE_COMMAND} -E rename ${FLYCV_SOURCE_DIR}/lib/${ANDROID_ABI}
+      ${FLYCV_LIB_DIR} && ${CMAKE_COMMAND} -E copy_directory
+      ${FLYCV_SOURCE_DIR}/include ${FLYCV_INC_DIR}
+    BUILD_BYPRODUCTS ${FLYCV_COMPILE_LIB})
+else()
+  ExternalProject_Add(
+    ${FLYCV_PROJECT}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    URL ${FLYCV_URL}
+    PREFIX ${FLYCV_PREFIX_DIR}
+    DOWNLOAD_NO_PROGRESS 1
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    UPDATE_COMMAND ""
+    INSTALL_COMMAND
+      ${CMAKE_COMMAND} -E remove_directory ${FLYCV_INSTALL_DIR} &&
+      ${CMAKE_COMMAND} -E make_directory ${FLYCV_INSTALL_DIR} &&
+      ${CMAKE_COMMAND} -E rename ${FLYCV_SOURCE_DIR}/lib/
+      ${FLYCV_LIB_DIR} && ${CMAKE_COMMAND} -E copy_directory
+      ${FLYCV_SOURCE_DIR}/include ${FLYCV_INC_DIR}
+    BUILD_BYPRODUCTS ${FLYCV_COMPILE_LIB})
+endif()
+
+add_library(external_flycv STATIC IMPORTED GLOBAL)
+set_property(TARGET external_flycv PROPERTY IMPORTED_LOCATION
+                                         ${FLYCV_COMPILE_LIB})
+add_dependencies(external_flycv ${FLYCV_PROJECT})
diff --git a/cmake/opencv.cmake b/cmake/opencv.cmake
@@ -184,7 +184,7 @@ else()
     if (WIN32)
       set(OpenCV_DIR ${OpenCV_DIR}/build)
     endif()
-    find_package(OpenCV REQUIRED PATHS ${OpenCV_DIR})
+    find_package(OpenCV REQUIRED PATHS ${OpenCV_DIR} NO_DEFAULT_PATH)
     include_directories(${OpenCV_INCLUDE_DIRS})
     list(APPEND DEPEND_LIBS opencv_core opencv_video opencv_highgui opencv_imgproc opencv_imgcodecs)
   endif()

diff --git a/cmake/paddlelite.cmake b/cmake/paddlelite.cmake
@@ -23,9 +23,15 @@ set(PADDLELITE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/${PADDLELITE_FILENAME})
 set(PADDLELITE_INC_DIR
     "${PADDLELITE_INSTALL_DIR}/include"
     CACHE PATH "paddlelite include directory." FORCE)
-set(PADDLELITE_LIB_DIR
-    "${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI}"
-    CACHE PATH "paddlelite lib directory." FORCE)
+if(ANDROID)    
+  set(PADDLELITE_LIB_DIR
+      "${PADDLELITE_INSTALL_DIR}/lib/${ANDROID_ABI}"
+      CACHE PATH "paddlelite lib directory." FORCE)
+else()
+  set(PADDLELITE_LIB_DIR
+  "${PADDLELITE_INSTALL_DIR}/lib/"
+  CACHE PATH "paddlelite lib directory." FORCE)
+endif()    
 set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLELITE_LIB_DIR}")
 
 set(PADDLELITE_URL_PREFIX "https://bj.bcebos.com/fastdeploy/third_libs")

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -37,6 +37,12 @@ function(config_fastdeploy_executable_link_flags TARGET_NAME)
   endif()
 endfunction()
 
+# Usage: add_fastdeploy_executable_cc_files(xxx_var vision detection)
+function(add_fastdeploy_executable_cc_files CC_FILES_VAR FIELD SUB_FIELD)
+  file(GLOB_RECURSE _EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/${FIELD}/${SUB_FIELD}/*/cpp/*.cc)
+  set(${CC_FILES_VAR} ${_EXAMPLE_SRCS} PARENT_SCOPE)
+endfunction()
+
 set(EXAMPLES_NUM 0)
 function(add_fastdeploy_executable FIELD CC_FILE)
   # temp target name/file var in function scope
@@ -66,15 +72,25 @@ endfunction()
 
 # vision examples
 if(BUILD_EXAMPLES AND ENABLE_VISION)
-  if(NOT ENABLE_VISION_VISUALIZE)
-    message(FATAL_ERROR "ENABLE_VISION_VISUALIZE must be ON while BUILD_EXAMPLES and ENABLE_VISION both ON.")
-  endif()
   if(EXISTS ${PROJECT_SOURCE_DIR}/examples/vision)
     message(STATUS "")
     message(STATUS "*************FastDeploy Vision Examples Summary**********")
-    file(GLOB_RECURSE ALL_VISION_EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/*/*/cpp/*.cc)
-    foreach(_CC_FILE ${ALL_VISION_EXAMPLE_SRCS})
-      add_fastdeploy_executable(vision ${_CC_FILE})
+    set(ALL_VISION_SUD_FIELDS classification
+                              detection
+                              facedet
+                              faceid
+                              keypointdetection
+                              matting
+                              ocr
+                              segmentation)
+    if(NOT ANDROID)
+      list(APPEND ALL_VISION_SUD_FIELDS tracking)
+    endif()
+    foreach(_SUB_FIELD ${ALL_VISION_SUD_FIELDS})
+      add_fastdeploy_executable_cc_files(_SUB_CC_FILES vision ${_SUB_FIELD})
+      foreach(_CC_FILE ${_SUB_CC_FILES})
+        add_fastdeploy_executable(vision ${_CC_FILE})
+      endforeach()
     endforeach()
     message(STATUS "  [FastDeploy Executable Path]        : ${EXECUTABLE_OUTPUT_PATH}")
   endif()

diff --git a/fastdeploy/core/config.h.in b/fastdeploy/core/config.h.in
@@ -49,8 +49,8 @@
 #cmakedefine ENABLE_VISION
 #endif
 
-#ifndef ENABLE_FALCONCV
-#cmakedefine ENABLE_FALCONCV
+#ifndef ENABLE_FLYCV
+#cmakedefine ENABLE_FLYCV
 #endif
 
 #ifndef ENABLE_TEXT
@@ -61,8 +61,10 @@
 #cmakedefine ENABLE_OPENCV_CUDA
 #endif
 
+#ifdef ENABLE_VISION
 #ifndef ENABLE_VISION_VISUALIZE
-#cmakedefine ENABLE_VISION_VISUALIZE
+#define ENABLE_VISION_VISUALIZE
+#endif
 #endif
 
 #ifndef ENABLE_FDTENSOR_FUNC

diff --git a/fastdeploy/vision/common/processors/base.cc b/fastdeploy/vision/common/processors/base.cc
@@ -28,8 +28,39 @@ bool Processor::operator()(Mat* mat, ProcLib lib) {
     target = default_lib;
   }
 
-  bool ret = ImplByOpenCV(mat);
-  return ret;
+  if (target == ProcLib::FLYCV) {
+#ifdef ENABLE_FLYCV
+    if (mat->mat_type != ProcLib::FLYCV) {
+      if (mat->layout != Layout::HWC) {
+        FDERROR << "Cannot convert cv::Mat to fcv::Mat while layout is not HWC." << std::endl;
+      }
+      fcv::Mat fcv_mat = ConvertOpenCVMatToFalconCV(*(mat->GetOpenCVMat()));
+      mat->SetMat(fcv_mat);
+    }
+    return ImplByFalconCV(mat);
+#else
+    FDASSERT(false, "FastDeploy didn't compile with FalconCV.");
+#endif
+  }
+  return ImplByOpenCV(mat);
+}
+
+void EnableFlyCV() {
+#ifdef ENABLE_FLYCV
+  Processor::default_lib = ProcLib::FLYCV;
+  FDINFO << "Will change to use image processing library "
+         << Processor::default_lib << std::endl;
+#else
+  FDWARNING << "FastDeploy didn't compile with FlyCV, "
+                "will fallback to use OpenCV instead."
+            << std::endl;
+#endif
+}
+
+void DisableFlyCV() {
+  Processor::default_lib = ProcLib::OPENCV;
+  FDINFO << "Will change to use image processing library "
+         << Processor::default_lib << std::endl;
 }
 
 }  // namespace vision

diff --git a/fastdeploy/vision/common/processors/base.h b/fastdeploy/vision/common/processors/base.h
@@ -22,7 +22,14 @@
 namespace fastdeploy {
 namespace vision {
 
-class Processor {
+/*! @brief Enable using FlyCV to process image while deploy vision models. Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will fallback to using OpenCV in other platform
+ */
+FASTDEPLOY_DECL void EnableFlyCV();
+
+/// Disable using FlyCV to process image while deploy vision models.
+FASTDEPLOY_DECL void DisableFlyCV();
+
+class FASTDEPLOY_DECL Processor {
  public:
   // default_lib has the highest priority
   // all the function in `processor` will force to use