From 672479de6d5b16ddba30c9aff0f7f87f74d53c2b Mon Sep 17 00:00:00 2001 From: DefTruth Date: Thu, 20 Oct 2022 14:58:31 +0800 Subject: [PATCH 01/10] [Backend] Add override flag to lite backend --- fastdeploy/backends/lite/lite_backend.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fastdeploy/backends/lite/lite_backend.h b/fastdeploy/backends/lite/lite_backend.h index fed33e28540..cf88390a433 100644 --- a/fastdeploy/backends/lite/lite_backend.h +++ b/fastdeploy/backends/lite/lite_backend.h @@ -56,14 +56,14 @@ class LiteBackend : public BaseBackend { const std::string& params_file, const LiteBackendOption& option = LiteBackendOption()); - bool Infer(std::vector& inputs, std::vector* outputs); + bool Infer(std::vector& inputs, std::vector* outputs) override; // NOLINT - int NumInputs() const { return inputs_desc_.size(); } + int NumInputs() const override { return inputs_desc_.size(); } - int NumOutputs() const { return outputs_desc_.size(); } + int NumOutputs() const override { return outputs_desc_.size(); } - TensorInfo GetInputInfo(int index); - TensorInfo GetOutputInfo(int index); + TensorInfo GetInputInfo(int index) override; + TensorInfo GetOutputInfo(int index) override; std::vector GetInputInfos() override; std::vector GetOutputInfos() override; From cda3e95aaf2a26e5af57f9af25823382aff95bfb Mon Sep 17 00:00:00 2001 From: DefTruth Date: Fri, 21 Oct 2022 13:02:35 +0800 Subject: [PATCH 02/10] [Docs] Add Android C++ SDK build docs --- CMakeLists.txt | 2 +- docs/cn/build_and_install/android.md | 103 +++++++++++++++++- .../download_prebuilt_libraries.md | 8 +- 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c57251d4262..2429a604667 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ option(WITH_TESTING "Whether to compile with unittest." OFF) option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF) option(WITH_LITE_STATIC "Use Paddle Lite static lib for Android." OFF) option(WITH_LITE_FULL_API "Use Paddle Lite full API lib for Android." ON) -option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." OFF) +option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." ON) # Please don't open this flag now, some bugs exists. # Only support Linux Now diff --git a/docs/cn/build_and_install/android.md b/docs/cn/build_and_install/android.md index c091b75d550..2421d3c66a4 100644 --- a/docs/cn/build_and_install/android.md +++ b/docs/cn/build_and_install/android.md @@ -1,3 +1,104 @@ # Android部署库编译 -进行中... +FastDeploy当前在Android仅支持Paddle-Lite后端推理,支持armeabi-v7a和arm64-v8a两种cpu架构,在armv8.2架构的arm设备支持fp16精度推理。相关编译选项说明如下: + +|编译选项|默认值|说明|备注| +|:---|:---|:---|:---| +|ENABLE_LITE_BACKEND|OFF|编译Android库时需要设置为ON| - | +|WITH_OPENCV_STATIC|OFF|是否使用OpenCV静态库| - | +|WITH_LITE_STATIC|OFF|是否使用Lite静态库| 暂不支持使用Lite静态库 | +|WITH_LITE_FULL_API|ON|是否使用Lite Full API库| 目前必须为ON | +|WITH_LITE_FP16|ON|是否使用带FP16支持的Lite库| 目前仅支持 arm64-v8a 架构| + +更多编译选项请参考[FastDeploy编译选项说明](./README.md) + +## Android C++ SDK 编译安装 + +编译需要满足: + +- Android SDK API >= 21 +- Android NDK >= 20 (当前仅支持clang编译工具链) +- cmake >= 3.10.0 + +编译前请先检查您的Android SDK 和 NDK 是否已经配置,如: +```bash +➜ echo $ANDROID_SDK +/Users/xxx/Library/Android/sdk +➜ echo $ANDROID_NDK +/Users/xxx/Library/Android/sdk/ndk/25.1.8937393 +``` +推荐使用 NDK>=20 进行交叉编译,编译命令如下: +```bash +# Download the latest source code +git clone https://github.com/PaddlePaddle/FastDeploy.git +cd FastDeploy + +# Setting up Android toolchanin +ANDROID_ABI=arm64-v8a # 'arm64-v8a', 'armeabi-v7a' +ANDROID_PLATFORM="android-21" # API >= 21 +ANDROID_STL=c++_shared # 'c++_shared', 'c++_static' +ANDROID_TOOLCHAIN=clang # 'clang' only +TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake + +# Create build directory +BUILD_ROOT=build/Android +BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-21 +FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install" +mkdir build && mkdir ${BUILD_ROOT} && mkdir ${BUILD_DIR} +cd ${BUILD_DIR} + +# Check fp16 support (only support arm64-v8a now) +WITH_LITE_FP16=ON +if [ "$ANDROID_ABI" = "armeabi-v7a" ]; then + WITH_LITE_FP16=OFF +fi + +# Cmake configuration with Android toolchain +cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DANDROID_ABI=${ANDROID_ABI} \ + -DANDROID_NDK=${ANDROID_NDK} \ + -DANDROID_PLATFORM=${ANDROID_PLATFORM} \ + -DANDROID_STL=${ANDROID_STL} \ + -DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} \ + -DENABLE_LITE_BACKEND=ON \ + -DENABLE_VISION=ON \ + -DWITH_LITE_FP16=${WITH_LITE_FP16} \ + -DCMAKE_INSTALL_PREFIX=${FASDEPLOY_INSTALL_DIR} \ + -Wno-dev ../../.. + +# Build FastDeploy Android C++ SDK +make -j8 +make install +``` +编译完成后,Android C++ SDK 保存在 `build/Android/arm64-v8a-api-21/install` 目录下,目录结构如下: +```bash +➜ tree . -d -L 3 +. +├── examples +├── include +│   └── fastdeploy # FastDeploy 头文件 +├── lib +│   └── arm64-v8a # FastDeploy Android 动态库 +└── third_libs # 第三方依赖库 + └── install + ├── opencv + └── paddlelite +``` +在examples/vision目录下可查看Android C++ SDK 使用案例: +```bash +. +├── classification +│   ├── paddleclas +│   │   ├── android # 图像分类Android使用案例 +│   │   ├── cpp +... +├── detection +│   ├── paddledetection +│   │   ├── android # 目标检测Android使用案例 +│   │   ├── cpp +... +``` +如何使用FastDeploy Android C++ SDK 请参考使用案例文档: +- [图像分类Android使用文档](../../../examples/vision/classification/paddleclas/android/README.md) +- [目标检测Android使用文档](../../../examples/vision/detection/paddledetection/android/README.md) diff --git a/docs/cn/build_and_install/download_prebuilt_libraries.md b/docs/cn/build_and_install/download_prebuilt_libraries.md index a320b692440..9182aa69fc7 100644 --- a/docs/cn/build_and_install/download_prebuilt_libraries.md +++ b/docs/cn/build_and_install/download_prebuilt_libraries.md @@ -17,7 +17,7 @@ FastDeploy提供各平台预编译库,供开发者直接下载安装使用。 Release版本(当前最新0.3.0)安装 ```bash -pip install fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html +pip install fastdeploy-gpu-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy.html ``` 其中推荐使用Conda配置开发环境 @@ -51,7 +51,7 @@ pip install fastdeploy-python -f https://www.paddlepaddle.org.cn/whl/fastdeploy. ## C++ SDK安装 -Release版本(当前最新0.3.0) +Release版本(当前最新0.3.0,Android为0.4.0 pre-release) | 平台 | 文件 | 说明 | | :--- | :--- | :---- | @@ -59,5 +59,5 @@ Release版本(当前最新0.3.0) | Windows x64 | [fastdeploy-win-x64-0.3.0.zip](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-win-x64-0.3.0.zip) | Visual Studio 16 2019编译产出 | | Mac OSX x64 | [fastdeploy-osx-x86_64-0.3.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-x86_64-0.3.0.tgz) | clang++ 10.0.0编译产出| | Mac OSX arm64 | [fastdeploy-osx-arm64-0.3.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-osx-arm64-0.3.0.tgz) | clang++ 13.0.0编译产出 | -| Linux aarch64 | [fastdeploy-linux-aarch64-0.3.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-aarch64-0.3.0.tgz) | g++ 6.3.0编译产出 | - +| Linux aarch64 | [fastdeploy-linux-aarch64-0.3.0.tgz](https://bj.bcebos.com/fastdeploy/release/cpp/fastdeploy-linux-aarch64-0.3.0.tgz) | g++ 6.3.0编译产出 | +| Android armv7&v8 | [fastdeploy-android-0.4.0-shared.tgz](https://bj.bcebos.com/fastdeploy/release/android/fastdeploy-android-0.4.0-shared.tgz) | NDK 25及clang++编译产出, 支持arm64-v8a及armeabi-v7a | From b97e2e24f5df3a01b8848664786c9e0bc49616d5 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Fri, 21 Oct 2022 13:58:16 +0800 Subject: [PATCH 03/10] [Doc] fix android_build_docs typos --- docs/cn/build_and_install/android.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cn/build_and_install/android.md b/docs/cn/build_and_install/android.md index 2421d3c66a4..0280df949ec 100644 --- a/docs/cn/build_and_install/android.md +++ b/docs/cn/build_and_install/android.md @@ -43,7 +43,7 @@ TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake # Create build directory BUILD_ROOT=build/Android BUILD_DIR=${BUILD_ROOT}/${ANDROID_ABI}-api-21 -FASDEPLOY_INSTALL_DIR="${ROOT_PATH}/${BUILD_DIR}/install" +FASDEPLOY_INSTALL_DIR="${BUILD_DIR}/install" mkdir build && mkdir ${BUILD_ROOT} && mkdir ${BUILD_DIR} cd ${BUILD_DIR} @@ -53,7 +53,7 @@ if [ "$ANDROID_ABI" = "armeabi-v7a" ]; then WITH_LITE_FP16=OFF fi -# Cmake configuration with Android toolchain +# CMake configuration with Android toolchain cmake -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \ -DCMAKE_BUILD_TYPE=MinSizeRel \ -DANDROID_ABI=${ANDROID_ABI} \ From 265f8ac64e5acfce07314512d7a2d74d7e313739 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:03:19 +0800 Subject: [PATCH 04/10] Update CMakeLists.txt --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2429a604667..c57251d4262 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ option(WITH_TESTING "Whether to compile with unittest." OFF) option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF) option(WITH_LITE_STATIC "Use Paddle Lite static lib for Android." OFF) option(WITH_LITE_FULL_API "Use Paddle Lite full API lib for Android." ON) -option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." ON) +option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." OFF) # Please don't open this flag now, some bugs exists. # Only support Linux Now From 3aca8c75a92e04f5a8d1cade1630a8ab33c1bfe8 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Fri, 21 Oct 2022 15:03:57 +0800 Subject: [PATCH 05/10] Update android.md --- docs/cn/build_and_install/android.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/build_and_install/android.md b/docs/cn/build_and_install/android.md index 0280df949ec..cdbdd588c82 100644 --- a/docs/cn/build_and_install/android.md +++ b/docs/cn/build_and_install/android.md @@ -8,7 +8,7 @@ FastDeploy当前在Android仅支持Paddle-Lite后端推理,支持armeabi-v7a |WITH_OPENCV_STATIC|OFF|是否使用OpenCV静态库| - | |WITH_LITE_STATIC|OFF|是否使用Lite静态库| 暂不支持使用Lite静态库 | |WITH_LITE_FULL_API|ON|是否使用Lite Full API库| 目前必须为ON | -|WITH_LITE_FP16|ON|是否使用带FP16支持的Lite库| 目前仅支持 arm64-v8a 架构| +|WITH_LITE_FP16|OFF|是否使用带FP16支持的Lite库| 目前仅支持 arm64-v8a 架构| 更多编译选项请参考[FastDeploy编译选项说明](./README.md) From 942c56eadfa3954bf11825d580d30dde420f4954 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Fri, 21 Oct 2022 20:44:56 +0800 Subject: [PATCH 06/10] [Doc] Add PicoDet Android demo docs --- docs/cn/faq/use_cpp_sdk_on_android.md | 4 + docs/cn/faq/use_java_sdk_on_android.md | 2 + .../paddledetection/android/README.md | 93 ++++++++++++++++++- 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 docs/cn/faq/use_cpp_sdk_on_android.md create mode 100644 docs/cn/faq/use_java_sdk_on_android.md diff --git a/docs/cn/faq/use_cpp_sdk_on_android.md b/docs/cn/faq/use_cpp_sdk_on_android.md new file mode 100644 index 00000000000..0d0bb3e252f --- /dev/null +++ b/docs/cn/faq/use_cpp_sdk_on_android.md @@ -0,0 +1,4 @@ +## 在 Android 中使用 FastDeploy C++ SDK +本文档将以PicoDet为例,讲解如何通过JNI,将FastDeploy中的模型封装到Android中进行调用。 + +- TODO diff --git a/docs/cn/faq/use_java_sdk_on_android.md b/docs/cn/faq/use_java_sdk_on_android.md new file mode 100644 index 00000000000..832ca1909c8 --- /dev/null +++ b/docs/cn/faq/use_java_sdk_on_android.md @@ -0,0 +1,2 @@ +## 在 Android 中使用 FastDeploy Java SDK +- TODO diff --git a/examples/vision/detection/paddledetection/android/README.md b/examples/vision/detection/paddledetection/android/README.md index 6a82dd18c1f..4ff2fd77a66 100644 --- a/examples/vision/detection/paddledetection/android/README.md +++ b/examples/vision/detection/paddledetection/android/README.md @@ -1,3 +1,92 @@ -## 目标检测 Android Demo 使用文档 +# 目标检测 PicoDet Android Demo 使用文档 -- TODO +在 Android 上实现实时的目标检测功能,此 Demo 有很好的的易用性和开放性,如在 Demo 中跑自己训练好的模型等。 + +## 环境准备 + +1. 在本地环境安装好 Android Studio 工具,详细安装方法请见[Android Stuido 官网](https://developer.android.com/studio)。 +2. 准备一部 Android 手机,并开启 USB 调试模式。开启方法: `手机设置 -> 查找开发者选项 -> 打开开发者选项和 USB 调试模式` + +**注意**:如果您的 Android Studio 尚未配置 NDK ,请根据 Android Studio 用户指南中的[安装及配置 NDK 和 CMake ](https://developer.android.com/studio/projects/install-ndk)内容,预先配置好 NDK 。您可以选择最新的 NDK 版本,或者使用 FastDeploy Android 预测库版本一样的 NDK + +## 部署步骤 + +1. 目标检测 PicoDet Demo 位于 `fastdeploy/examples/vision/detection/paddledetection/android` 目录 +2. 用 Android Studio 打开 paddledetection/android 工程 +3. 手机连接电脑,打开 USB 调试和文件传输模式,并在 Android Studio 上连接自己的手机设备(手机需要开启允许从 USB 安装软件权限) + +

+image +

+ +> **注意:** +>> 如果您在导入项目、编译或者运行过程中遇到 NDK 配置错误的提示,请打开 ` File > Project Structure > SDK Location`,修改 `Andriod NDK location` 为您本机配置的 NDK 所在路径。本工程默认使用的NDK版本为20. +>> 如果您是通过 Andriod Studio 的 SDK Tools 下载的 NDK (见本章节"环境准备"),可以直接点击下拉框选择默认路径。 +>> 还有一种 NDK 配置方法,你可以在 `paddledetection/android/local.properties` 文件中手动完成 NDK 路径配置,如下图所示 +>> 如果以上步骤仍旧无法解决 NDK 配置错误,请尝试根据 Andriod Studio 官方文档中的[更新 Android Gradle 插件](https://developer.android.com/studio/releases/gradle-plugin?hl=zh-cn#updating-plugin)章节,尝试更新Android Gradle plugin版本。 + +4. 点击 Run 按钮,自动编译 APP 并安装到手机。(该过程会自动下载预编译的 FastDeploy Android 库,需要联网) +成功后效果如下,图一:APP 安装到手机 图二: APP 打开后的效果,会自动识别图片中的物体并标记 + + | APP 图标 | APP 效果 | + | --- | --- | + | ![app_pic ](https://user-images.githubusercontent.com/31974251/197170082-a2bdd49d-60ea-4df0-af63-18ed898a746e.jpg) | ![app_res](https://user-images.githubusercontent.com/31974251/197169609-bb214af3-d6e7-4433-bb96-1225cddd441c.jpg) | + +## PicoDet Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式,方式一是通过构造函数直接初始化;方式二是,通过调用init函数进行初始化。初始化参数说明: + - modelFile: String, paddle格式的模型文件路径,如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径,如 model.pdiparams + - configFile: String, 模型推理的预处理配置文件,如 infer_cfg.yml + - labelFile: String, 可选参数,表示label标签文件所在路径,用于可视化 + - option: RuntimeOption,模型初始化option + +```java +// 构造函数: constructor w/o label file +public PicoDet(); // 空构造函数,之后可以调用init初始化 +public PicoDet(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public PicoDet(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public boolean init(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +``` +- 模型预测 API:模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指,不保存图片以及不渲染结果到Bitmap上,仅预测推理结果。预测并且可视化是指,预测结果以及可视化,并将可视化后的图片保存到指定的途径,以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测:不保存图片以及不渲染结果到Bitmap上 +public DetectionResult predict(Bitmap ARGB8888Bitmap); +// 预测并且可视化:预测结果以及可视化,并将可视化后的图片保存到指定的途径,以及将可视化结果渲染在Bitmap上 +public DetectionResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold) +``` +- 模型资源释放 API:调用 release() API 可以释放模型资源,返回true表示释放成功,false表示失败;调用 initialized() 可以判断模型是否初始化成功,true表示初始化成功,false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` +- RuntimeOption设置说明 +```java +public void enableLiteFp16(); // 开启fp16精度推理 +public void disableLiteFP16(); // 关闭fp16精度推理 +public void setCpuThreadNum(int threadNum); // 设置线程数 +public void setLitePowerMode(LitePowerMode mode); // 设置能耗模式 +public void setLitePowerMode(String modeStr); // 通过字符串形式设置能耗模式 +public void enableRecordTimeOfRuntime(); // 是否打印模型运行耗时 +``` + +- 模型结果DetectionResult说明 +```java +// Not support MaskRCNN now. +public float[][] mBoxes; // [n,4] 检测框 (x1,y1,x2,y2) +public float[] mScores; // [n] 得分 +public int[] mLabelIds; // [n] 分类ID +public boolean initialized(); // 检测结果是否有效 +``` + + +## PicoDet Android Demo 工程简介 + +## 替换 FastDeploy 预测库和模型 + +## 如何通过 JNI 在 Native 层接入 FastDeploy C++ API ? + +如果您对如何通过JNI来接入FastDeploy C++ API感兴趣,可以参考以下内容: +- [app/src/main/cpp 代码实现](./app/src/main/cpp/) +- [在 Android 中使用 FastDeploy C++ SDK](../../../../../docs/cn/faq/use_cpp_sdk_on_android.md) From 92173018a47c1fec95eb57278e4cafde3c39b0fe Mon Sep 17 00:00:00 2001 From: DefTruth Date: Sat, 22 Oct 2022 18:08:58 +0800 Subject: [PATCH 07/10] [Doc] Update PicoDet Andorid demo docs --- .../paddledetection/android/README.md | 82 ++++++++++++++++--- .../fastdeploy/vision/detection/PicoDet.java | 14 ++++ 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/examples/vision/detection/paddledetection/android/README.md b/examples/vision/detection/paddledetection/android/README.md index 4ff2fd77a66..9cde9bd9f91 100644 --- a/examples/vision/detection/paddledetection/android/README.md +++ b/examples/vision/detection/paddledetection/android/README.md @@ -26,23 +26,25 @@ >> 如果以上步骤仍旧无法解决 NDK 配置错误,请尝试根据 Andriod Studio 官方文档中的[更新 Android Gradle 插件](https://developer.android.com/studio/releases/gradle-plugin?hl=zh-cn#updating-plugin)章节,尝试更新Android Gradle plugin版本。 4. 点击 Run 按钮,自动编译 APP 并安装到手机。(该过程会自动下载预编译的 FastDeploy Android 库,需要联网) -成功后效果如下,图一:APP 安装到手机 图二: APP 打开后的效果,会自动识别图片中的物体并标记 +成功后效果如下,图一:APP 安装到手机;图二: APP 打开后的效果,会自动识别图片中的物体并标记;图三:APP设置选项,点击右上角的设置图片,可以设置不同选项进行体验。 - | APP 图标 | APP 效果 | - | --- | --- | - | ![app_pic ](https://user-images.githubusercontent.com/31974251/197170082-a2bdd49d-60ea-4df0-af63-18ed898a746e.jpg) | ![app_res](https://user-images.githubusercontent.com/31974251/197169609-bb214af3-d6e7-4433-bb96-1225cddd441c.jpg) | + | APP 图标 | APP 效果 | APP设置项 + | --- | --- | --- | + | ![app_pic ](https://user-images.githubusercontent.com/31974251/197170082-a2bdd49d-60ea-4df0-af63-18ed898a746e.jpg) | ![app_res](https://user-images.githubusercontent.com/31974251/197169609-bb214af3-d6e7-4433-bb96-1225cddd441c.jpg) | ![app_setup](https://user-images.githubusercontent.com/31974251/197332983-afbfa6d5-4a3b-4c54-a528-4a3e58441be1.jpg) | ## PicoDet Java API 说明 -- 模型初始化 API: 模型初始化API包含两种方式,方式一是通过构造函数直接初始化;方式二是,通过调用init函数进行初始化。初始化参数说明: +- 模型初始化 API: 模型初始化API包含两种方式,方式一是通过构造函数直接初始化;方式二是,通过调用init函数,在合适的程序节点进行初始化。PicoDet初始化参数说明如下: - modelFile: String, paddle格式的模型文件路径,如 model.pdmodel - paramFile: String, paddle格式的参数文件路径,如 model.pdiparams - configFile: String, 模型推理的预处理配置文件,如 infer_cfg.yml - - labelFile: String, 可选参数,表示label标签文件所在路径,用于可视化 - - option: RuntimeOption,模型初始化option + - labelFile: String, 可选参数,表示label标签文件所在路径,用于可视化,如 coco_label_list.txt,每一行包含一个label + - option: RuntimeOption,可选参数,模型初始化option。如果不传入该参数则会使用默认的运行时选项。 ```java // 构造函数: constructor w/o label file public PicoDet(); // 空构造函数,之后可以调用init初始化 +public PicoDet(String modelFile, String paramsFile, String configFile); +public PicoDet(String modelFile, String paramsFile, String configFile, String labelFile); public PicoDet(String modelFile, String paramsFile, String configFile, RuntimeOption option); public PicoDet(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); // 手动调用init初始化: call init manually w/o label file @@ -73,20 +75,78 @@ public void enableRecordTimeOfRuntime(); // 是否打印模型运行耗时 - 模型结果DetectionResult说明 ```java -// Not support MaskRCNN now. public float[][] mBoxes; // [n,4] 检测框 (x1,y1,x2,y2) public float[] mScores; // [n] 得分 public int[] mLabelIds; // [n] 分类ID public boolean initialized(); // 检测结果是否有效 -``` +``` + +- 模型调用示例1:使用构造函数以及默认的RuntimeOption +```java +import java.nio.ByteBuffer; +import android.graphics.Bitmap; +import android.opengl.GLES20; + +import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; + +// 初始化模型 +PicoDet model = new PicoDet("picodet_s_320_coco_lcnet/model.pdmodel", + "picodet_s_320_coco_lcnet/model.pdiparams", + "picodet_s_320_coco_lcnet/infer_cfg.yml"); + +// 读取图片: 以下仅为读取Bitmap的伪代码 +ByteBuffer pixelBuffer = ByteBuffer.allocate(width * height * 4); +GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuffer); +Bitmap ARGB8888ImageBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); +ARGB8888ImageBitmap.copyPixelsFromBuffer(pixelBuffer); +// 模型推理 +DetectionResult result = model.predict(ARGB8888ImageBitmap); -## PicoDet Android Demo 工程简介 +// 释放模型资源 +model.release(); +``` + +- 模型调用示例2: 在合适的程序节点,手动调用init,并自定义RuntimeOption +```java +// import 同上 ... +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.LitePowerMode; +import com.baidu.paddle.fastdeploy.vision.DetectionResult; +import com.baidu.paddle.fastdeploy.vision.detection.PicoDet; +// 新建空模型 +PicoDet model = new PicoDet(); +// 模型路径 +String modelFile = "picodet_s_320_coco_lcnet/model.pdmodel"; +String paramFile = "picodet_s_320_coco_lcnet/model.pdiparams"; +String configFile = "picodet_s_320_coco_lcnet/infer_cfg.yml"; +// 指定RuntimeOption +RuntimeOption option = new RuntimeOption(); +option.setCpuThreadNum(2); +option.setLitePowerMode(LitePowerMode.LITE_POWER_HIGH); +option.enableRecordTimeOfRuntime(); +option.enableLiteFp16(); +// 使用init函数初始化 +model.init(modelFile, paramFile, configFile, option); +// Bitmap读取、模型预测、资源释放 同上 ... +``` +更详细的用法请参考 [MainActivity](./app/src/main/java/com/baidu/paddle/fastdeploy/examples/MainActivity.java#L207) 中的用法 ## 替换 FastDeploy 预测库和模型 +替换FastDeploy预测库和模型的步骤非常简单。预测库所在的位置为 `app/libs/fastdeploy-android-xxx-shared`,其中 `xxx` 表示当前您使用的预测库版本号。模型所在的位置为,`app/src/main/assets/models/picodet_s_320_coco_lcnet`。 +- 替换FastDeploy预测库的步骤:(1)下载或编译最新的FastDeploy Android预测库,解压缩后放在 `app/libs` 目录下;(2)修改 `app/src/main/cpp/CMakeLists.txt` 中的预测库路径,指向您下载或编译的预测库路径。如: +```cmake +set(FastDeploy_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../libs/fastdeploy-android-xxx-shared") +``` +- 替换PicoDet模型的步骤:(1)将您的PicoDet模型放在 `app/src/main/assets/models` 目录下;(2)修改 `app/src/main/res/values/strings.xml` 中模型路径的默认值,如: +```xml + +models/picodet_s_320_coco_lcnet +labels/coco_label_list.txt +``` ## 如何通过 JNI 在 Native 层接入 FastDeploy C++ API ? - 如果您对如何通过JNI来接入FastDeploy C++ API感兴趣,可以参考以下内容: - [app/src/main/cpp 代码实现](./app/src/main/cpp/) - [在 Android 中使用 FastDeploy C++ SDK](../../../../../docs/cn/faq/use_cpp_sdk_on_android.md) diff --git a/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java b/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java index bbd52eecee7..9729eeb8a43 100644 --- a/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java +++ b/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java @@ -14,6 +14,20 @@ public PicoDet() { mInitialized = false; } + // Constructor with default runtime option + public PicoDet(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, "", new RuntimeOption()); + } + + public PicoDet(String modelFile, + String paramsFile, + String configFile, + String labelFile) { + init_(modelFile, paramsFile, configFile, labelFile, new RuntimeOption()); + } + // Constructor without label file public PicoDet(String modelFile, String paramsFile, From c9eed32e8edb5749ef482bd964a604c69d0e9354 Mon Sep 17 00:00:00 2001 From: DefTruth Date: Sat, 22 Oct 2022 20:41:56 +0800 Subject: [PATCH 08/10] [Doc] Update PaddleClasModel Android demo docs --- .../paddleclas/android/README.md | 156 +++++++++++++++++- .../classification/PaddleClasModel.java | 14 ++ .../fastdeploy/vision/detection/PicoDet.java | 14 ++ .../paddledetection/android/README.md | 8 +- .../classification/PaddleClasModel.java | 14 ++ 5 files changed, 202 insertions(+), 4 deletions(-) diff --git a/examples/vision/classification/paddleclas/android/README.md b/examples/vision/classification/paddleclas/android/README.md index 5a595241daf..cd1579c846c 100644 --- a/examples/vision/classification/paddleclas/android/README.md +++ b/examples/vision/classification/paddleclas/android/README.md @@ -1,3 +1,155 @@ -## 图像分类 Android Demo 使用文档 +## 图像分类 PaddleClas Android Demo 使用文档 -- TODO +在 Android 上实现实时的图像分类功能,此 Demo 有很好的的易用性和开放性,如在 Demo 中跑自己训练好的模型等。 + +## 环境准备 + +1. 在本地环境安装好 Android Studio 工具,详细安装方法请见[Android Stuido 官网](https://developer.android.com/studio)。 +2. 准备一部 Android 手机,并开启 USB 调试模式。开启方法: `手机设置 -> 查找开发者选项 -> 打开开发者选项和 USB 调试模式` + +**注意**:如果您的 Android Studio 尚未配置 NDK ,请根据 Android Studio 用户指南中的[安装及配置 NDK 和 CMake ](https://developer.android.com/studio/projects/install-ndk)内容,预先配置好 NDK 。您可以选择最新的 NDK 版本,或者使用 FastDeploy Android 预测库版本一样的 NDK + +## 部署步骤 + +1. 目标检测 PaddleClas Demo 位于 `fastdeploy/examples/vision/classification/paddleclas/android` 目录 +2. 用 Android Studio 打开 paddleclas/android 工程 +3. 手机连接电脑,打开 USB 调试和文件传输模式,并在 Android Studio 上连接自己的手机设备(手机需要开启允许从 USB 安装软件权限) + +

+image +

+ +> **注意:** +>> 如果您在导入项目、编译或者运行过程中遇到 NDK 配置错误的提示,请打开 ` File > Project Structure > SDK Location`,修改 `Andriod NDK location` 为您本机配置的 NDK 所在路径。本工程默认使用的NDK版本为20. +>> 如果您是通过 Andriod Studio 的 SDK Tools 下载的 NDK (见本章节"环境准备"),可以直接点击下拉框选择默认路径。 +>> 还有一种 NDK 配置方法,你可以在 `paddleclas/android/local.properties` 文件中手动完成 NDK 路径配置,如下图所示 +>> 如果以上步骤仍旧无法解决 NDK 配置错误,请尝试根据 Andriod Studio 官方文档中的[更新 Android Gradle 插件](https://developer.android.com/studio/releases/gradle-plugin?hl=zh-cn#updating-plugin)章节,尝试更新Android Gradle plugin版本。 + +4. 点击 Run 按钮,自动编译 APP 并安装到手机。(该过程会自动下载预编译的 FastDeploy Android 库,需要联网) +成功后效果如下,图一:APP 安装到手机;图二: APP 打开后的效果,会自动识别图片中的物体并标记;图三:APP设置选项,点击右上角的设置图片,可以设置不同选项进行体验。 + + | APP 图标 | APP 效果 | APP设置项 + | --- | --- | --- | + | ![app_pic ](https://user-images.githubusercontent.com/31974251/197170082-a2bdd49d-60ea-4df0-af63-18ed898a746e.jpg) | ![app_res](https://user-images.githubusercontent.com/31974251/197339363-ae7acd5d-88b8-4365-aea5-b27826c6a25f.jpg) | ![app_setup](https://user-images.githubusercontent.com/31974251/197339378-bb30b108-2d77-4b30-981d-d687b6fca8f6.jpg) | + +## PaddleClasModel Java API 说明 +- 模型初始化 API: 模型初始化API包含两种方式,方式一是通过构造函数直接初始化;方式二是,通过调用init函数,在合适的程序节点进行初始化。PaddleClasModel初始化参数说明如下: + - modelFile: String, paddle格式的模型文件路径,如 model.pdmodel + - paramFile: String, paddle格式的参数文件路径,如 model.pdiparams + - configFile: String, 模型推理的预处理配置文件,如 infer_cfg.yml + - labelFile: String, 可选参数,表示label标签文件所在路径,用于可视化,如 imagenet1k_label_list.txt,每一行包含一个label + - option: RuntimeOption,可选参数,模型初始化option。如果不传入该参数则会使用默认的运行时选项。 + +```java +// 构造函数: constructor w/o label file +public PaddleClasModel(); // 空构造函数,之后可以调用init初始化 +public PaddleClasModel(String modelFile, String paramsFile, String configFile); +public PaddleClasModel(String modelFile, String paramsFile, String configFile, String labelFile); +public PaddleClasModel(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public PaddleClasModel(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +// 手动调用init初始化: call init manually w/o label file +public boolean init(String modelFile, String paramsFile, String configFile, RuntimeOption option); +public boolean init(String modelFile, String paramsFile, String configFile, String labelFile, RuntimeOption option); +``` +- 模型预测 API:模型预测API包含直接预测的API以及带可视化功能的API。直接预测是指,不保存图片以及不渲染结果到Bitmap上,仅预测推理结果。预测并且可视化是指,预测结果以及可视化,并将可视化后的图片保存到指定的途径,以及将可视化结果渲染在Bitmap(目前支持ARGB8888格式的Bitmap), 后续可将该Bitmap在camera中进行显示。 +```java +// 直接预测:不保存图片以及不渲染结果到Bitmap上 +public ClassifyResult predict(Bitmap ARGB8888Bitmap); +// 预测并且可视化:预测结果以及可视化,并将可视化后的图片保存到指定的途径,以及将可视化结果渲染在Bitmap上 +public ClassifyResult predict(Bitmap ARGB8888Bitmap, String savedImagePath, float scoreThreshold) +``` +- 模型资源释放 API:调用 release() API 可以释放模型资源,返回true表示释放成功,false表示失败;调用 initialized() 可以判断模型是否初始化成功,true表示初始化成功,false表示失败。 +```java +public boolean release(); // 释放native资源 +public boolean initialized(); // 检查是否初始化成功 +``` +- RuntimeOption设置说明 +```java +public void enableLiteFp16(); // 开启fp16精度推理 +public void disableLiteFP16(); // 关闭fp16精度推理 +public void setCpuThreadNum(int threadNum); // 设置线程数 +public void setLitePowerMode(LitePowerMode mode); // 设置能耗模式 +public void setLitePowerMode(String modeStr); // 通过字符串形式设置能耗模式 +public void enableRecordTimeOfRuntime(); // 是否打印模型运行耗时 +``` + +- 模型结果ClassifyResult说明 +```java +public float[] mScores; // [n] 得分 +public int[] mLabelIds; // [n] 分类ID +public boolean initialized(); // 检测结果是否有效 +``` + +- 模型调用示例1:使用构造函数以及默认的RuntimeOption +```java +import java.nio.ByteBuffer; +import android.graphics.Bitmap; +import android.opengl.GLES20; + +import com.baidu.paddle.fastdeploy.vision.ClassifyResult; +import com.baidu.paddle.fastdeploy.vision.classification.PaddleClasModel; + +// 初始化模型 +PaddleClasModel model = new PaddleClasModel("MobileNetV1_x0_25_infer/inference.pdmodel", + "MobileNetV1_x0_25_infer/inference.pdiparams", + "MobileNetV1_x0_25_infer/inference_cls.yml"); + +// 读取图片: 以下仅为读取Bitmap的伪代码 +ByteBuffer pixelBuffer = ByteBuffer.allocate(width * height * 4); +GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, pixelBuffer); +Bitmap ARGB8888ImageBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); +ARGB8888ImageBitmap.copyPixelsFromBuffer(pixelBuffer); + +// 模型推理 +ClassifyResult result = model.predict(ARGB8888ImageBitmap); + +// 释放模型资源 +model.release(); +``` + +- 模型调用示例2: 在合适的程序节点,手动调用init,并自定义RuntimeOption +```java +// import 同上 ... +import com.baidu.paddle.fastdeploy.RuntimeOption; +import com.baidu.paddle.fastdeploy.LitePowerMode; +import com.baidu.paddle.fastdeploy.vision.ClassifyResult; +import com.baidu.paddle.fastdeploy.vision.classification.PaddleClasModel; +// 新建空模型 +PaddleClasModel model = new PaddleClasModel(); +// 模型路径 +String modelFile = "MobileNetV1_x0_25_infer/inference.pdmodel"; +String paramFile = "MobileNetV1_x0_25_infer/inference.pdiparams"; +String configFile = "MobileNetV1_x0_25_infer/inference_cls.yml"; +// 指定RuntimeOption +RuntimeOption option = new RuntimeOption(); +option.setCpuThreadNum(2); +option.setLitePowerMode(LitePowerMode.LITE_POWER_HIGH); +option.enableRecordTimeOfRuntime(); +option.enableLiteFp16(); +// 使用init函数初始化 +model.init(modelFile, paramFile, configFile, option); +// Bitmap读取、模型预测、资源释放 同上 ... +``` +更详细的用法请参考 [MainActivity](./app/src/main/java/com/baidu/paddle/fastdeploy/examples/MainActivity.java#L207) 中的用法 + +## 替换 FastDeploy 预测库和模型 +替换FastDeploy预测库和模型的步骤非常简单。预测库所在的位置为 `app/libs/fastdeploy-android-xxx-shared`,其中 `xxx` 表示当前您使用的预测库版本号。模型所在的位置为,`app/src/main/assets/models/MobileNetV1_x0_25_infer`。 +- 替换FastDeploy预测库的步骤: + - 下载或编译最新的FastDeploy Android预测库,解压缩后放在 `app/libs` 目录下; + - 修改 `app/src/main/cpp/CMakeLists.txt` 中的预测库路径,指向您下载或编译的预测库路径。如: +```cmake +set(FastDeploy_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../libs/fastdeploy-android-xxx-shared") +``` +- 替换PaddleClas模型的步骤: + - 将您的PaddleClas分类模型放在 `app/src/main/assets/models` 目录下; + - 修改 `app/src/main/res/values/strings.xml` 中模型路径的默认值,如: +```xml + +models/MobileNetV1_x0_25_infer +labels/imagenet1k_label_list.txt +``` + +## 如何通过 JNI 在 Native 层接入 FastDeploy C++ API ? +如果您对如何通过JNI来接入FastDeploy C++ API感兴趣,可以参考以下内容: +- [app/src/main/cpp 代码实现](./app/src/main/cpp/) +- [在 Android 中使用 FastDeploy C++ SDK](../../../../../docs/cn/faq/use_cpp_sdk_on_android.md) diff --git a/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java b/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java index f0524604e05..b4a56e30955 100644 --- a/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java +++ b/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java @@ -14,6 +14,20 @@ public PaddleClasModel() { mInitialized = false; } + // Constructor with default runtime option + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, "", new RuntimeOption()); + } + + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile, + String labelFile) { + init_(modelFile, paramsFile, configFile, labelFile, new RuntimeOption()); + } + // Constructor without label file public PaddleClasModel(String modelFile, String paramsFile, diff --git a/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java b/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java index bbd52eecee7..9729eeb8a43 100644 --- a/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java +++ b/examples/vision/classification/paddleclas/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java @@ -14,6 +14,20 @@ public PicoDet() { mInitialized = false; } + // Constructor with default runtime option + public PicoDet(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, "", new RuntimeOption()); + } + + public PicoDet(String modelFile, + String paramsFile, + String configFile, + String labelFile) { + init_(modelFile, paramsFile, configFile, labelFile, new RuntimeOption()); + } + // Constructor without label file public PicoDet(String modelFile, String paramsFile, diff --git a/examples/vision/detection/paddledetection/android/README.md b/examples/vision/detection/paddledetection/android/README.md index 9cde9bd9f91..4a57f98f953 100644 --- a/examples/vision/detection/paddledetection/android/README.md +++ b/examples/vision/detection/paddledetection/android/README.md @@ -135,11 +135,15 @@ model.init(modelFile, paramFile, configFile, option); ## 替换 FastDeploy 预测库和模型 替换FastDeploy预测库和模型的步骤非常简单。预测库所在的位置为 `app/libs/fastdeploy-android-xxx-shared`,其中 `xxx` 表示当前您使用的预测库版本号。模型所在的位置为,`app/src/main/assets/models/picodet_s_320_coco_lcnet`。 -- 替换FastDeploy预测库的步骤:(1)下载或编译最新的FastDeploy Android预测库,解压缩后放在 `app/libs` 目录下;(2)修改 `app/src/main/cpp/CMakeLists.txt` 中的预测库路径,指向您下载或编译的预测库路径。如: +- 替换FastDeploy预测库的步骤: + - 下载或编译最新的FastDeploy Android预测库,解压缩后放在 `app/libs` 目录下; + - 修改 `app/src/main/cpp/CMakeLists.txt` 中的预测库路径,指向您下载或编译的预测库路径。如: ```cmake set(FastDeploy_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../libs/fastdeploy-android-xxx-shared") ``` -- 替换PicoDet模型的步骤:(1)将您的PicoDet模型放在 `app/src/main/assets/models` 目录下;(2)修改 `app/src/main/res/values/strings.xml` 中模型路径的默认值,如: +- 替换PicoDet模型的步骤: + - 将您的PicoDet模型放在 `app/src/main/assets/models` 目录下; + - 修改 `app/src/main/res/values/strings.xml` 中模型路径的默认值,如: ```xml models/picodet_s_320_coco_lcnet diff --git a/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java b/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java index f0524604e05..b4a56e30955 100644 --- a/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java +++ b/examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/classification/PaddleClasModel.java @@ -14,6 +14,20 @@ public PaddleClasModel() { mInitialized = false; } + // Constructor with default runtime option + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile) { + init_(modelFile, paramsFile, configFile, "", new RuntimeOption()); + } + + public PaddleClasModel(String modelFile, + String paramsFile, + String configFile, + String labelFile) { + init_(modelFile, paramsFile, configFile, labelFile, new RuntimeOption()); + } + // Constructor without label file public PaddleClasModel(String modelFile, String paramsFile, From d37e626ca0c1347e5b827246deaf2395a0f8718a Mon Sep 17 00:00:00 2001 From: DefTruth Date: Sat, 22 Oct 2022 21:43:01 +0800 Subject: [PATCH 09/10] [Doc] Update fastdeploy android jni docs --- docs/cn/build_and_install/android.md | 1 + docs/cn/faq/use_cpp_sdk_on_android.md | 259 +++++++++++++++++- .../paddleclas/android/README.md | 2 +- 3 files changed, 258 insertions(+), 4 deletions(-) diff --git a/docs/cn/build_and_install/android.md b/docs/cn/build_and_install/android.md index cdbdd588c82..899ec2985a5 100644 --- a/docs/cn/build_and_install/android.md +++ b/docs/cn/build_and_install/android.md @@ -102,3 +102,4 @@ make install 如何使用FastDeploy Android C++ SDK 请参考使用案例文档: - [图像分类Android使用文档](../../../examples/vision/classification/paddleclas/android/README.md) - [目标检测Android使用文档](../../../examples/vision/detection/paddledetection/android/README.md) +- [在 Android 通过 JNI 中使用 FastDeploy C++ SDK](../../../../../docs/cn/faq/use_cpp_sdk_on_android.md) diff --git a/docs/cn/faq/use_cpp_sdk_on_android.md b/docs/cn/faq/use_cpp_sdk_on_android.md index 0d0bb3e252f..dc08297dd83 100644 --- a/docs/cn/faq/use_cpp_sdk_on_android.md +++ b/docs/cn/faq/use_cpp_sdk_on_android.md @@ -1,4 +1,257 @@ -## 在 Android 中使用 FastDeploy C++ SDK -本文档将以PicoDet为例,讲解如何通过JNI,将FastDeploy中的模型封装到Android中进行调用。 +# 在 Android 中通过 JNI 使用 FastDeploy C++ SDK +本文档将以PicoDet为例,讲解如何通过JNI,将FastDeploy中的模型封装到Android中进行调用。阅读本文档,您至少需要了解C++、Java、JNI以及Android的基础知识。如果您主要关注如何在Java层如何调用FastDeploy的API,则可以不阅读本文档。 -- TODO +## 新建PicoDet Java类及定义需要C++层实现的native API +```java +public class PicoDet { + protected long mNativeModelContext = 0; // Context from native. + protected boolean mInitialized = false; + // ... + // Bind predictor from native context. + private static native long bindNative(String modelFile, + String paramsFile, + String configFile, + int cpuNumThread, + boolean enableLiteFp16, + int litePowerMode, + String liteOptimizedModelDir, + boolean enableRecordTimeOfRuntime, + String labelFile); + + // Call prediction from native context. + private static native long predictNative(long nativeModelContext, + Bitmap ARGB8888Bitmap, + boolean saved, + String savedImagePath, + float scoreThreshold, + boolean rendering); + + // Release buffers allocated in native context. + private static native boolean releaseNative(long nativeModelContext); + + // Initializes at the beginning. + static { + FastDeployInitializer.init(); + } +} +``` +这些被标记为native的接口是需要通过JNI的方式实现,并在Java层供PicoDet类调用。完整的PicoDet Java代码请参考 [PicoDet.java](../../../examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java) 。各个函数说明如下: +- bindNative: C++层初始化模型资源,如果成功初始化,则返回指向该模型的指针(long类型),否则返回0指针 +- predictNative: 通过已经初始化好的模型指针,在C++层执行预测代码,如果预测成功则返回指向预测结果的指针,否则返回0指针。注意,该结果指针在当次预测使用完之后需要释放,具体操作请参考 [PicoDet.java](../../../examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java) 中的predict函数。 +- releaseNative: 根据传入的模型指针,在C++层释放模型资源。 + +## Android Studio 生成JNI函数定义 +- Android Studio 生成 JNI 函数定义: 鼠标停留在Java中定义的native函数上,Android Studio 便会提示是否要创建JNI函数定义;这里,我们把JNI函数定义创建在一个事先创建好的c++文件`picodet_jni.cc`上; + +使用Android Studio创建JNI函数定义: +![](https://user-images.githubusercontent.com/31974251/197341065-cdf8f626-4bb1-4a57-8d7a-80b382fe994e.png) + +将JNI函数定义创建在picodet_jni.cc上: +![](https://user-images.githubusercontent.com/31974251/197341190-b887dec5-fa75-43c9-9ab3-7ead50c0eb45.png) + +创建的JNI函数定义如下: +![](https://user-images.githubusercontent.com/31974251/197341274-e9671bac-9e77-4043-a870-9d5db914586b.png) + +其他native函数对应的JNI函数定义的创建和此流程一样。 + +## 在C++层实现JNI函数 +以下为PicoDet JNI层实现的示例,相关的辅助函数不在此处赘述,完整的C++代码请参考 [android/app/src/main/cpp](../../../examples/vision/detection/paddledetection/android/app/src/main/cpp/). +```C++ +#include // NOLINT + +#include "fastdeploy_jni.h" // NOLINT + +#ifdef __cplusplus +extern "C" { +#endif + +// 绑定C++层的模型 +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_bindNative( + JNIEnv *env, jclass clazz, jstring model_file, jstring params_file, + jstring config_file, jint cpu_num_thread, jboolean enable_lite_fp16, + jint lite_power_mode, jstring lite_optimized_model_dir, + jboolean enable_record_time_of_runtime, jstring label_file) { + std::string c_model_file = fastdeploy::jni::ConvertTo(env, model_file); + std::string c_params_file = fastdeploy::jni::ConvertTo(env, params_file); + std::string c_config_file = astdeploy::jni::ConvertTo(env, config_file); + std::string c_label_file = fastdeploy::jni::ConvertTo(env, label_file); + std::string c_lite_optimized_model_dir = fastdeploy::jni::ConvertTo(env, lite_optimized_model_dir); + auto c_cpu_num_thread = static_cast(cpu_num_thread); + auto c_enable_lite_fp16 = static_cast(enable_lite_fp16); + auto c_lite_power_mode = static_cast(lite_power_mode); + fastdeploy::RuntimeOption c_option; + c_option.UseCpu(); + c_option.UseLiteBackend(); + c_option.SetCpuThreadNum(c_cpu_num_thread); + c_option.SetLitePowerMode(c_lite_power_mode); + c_option.SetLiteOptimizedModelDir(c_lite_optimized_model_dir); + if (c_enable_lite_fp16) { + c_option.EnableLiteFP16(); + } + // 如果您实现的是其他模型,比如PPYOLOE,请注意修改此处绑定的C++类型 + auto c_model_ptr = new fastdeploy::vision::detection::PicoDet( + c_model_file, c_params_file, c_config_file, c_option); + // Enable record Runtime time costs. + if (enable_record_time_of_runtime) { + c_model_ptr->EnableRecordTimeOfRuntime(); + } + // Load detection labels if label path is not empty. + if ((!fastdeploy::jni::AssetsLoaderUtils::IsDetectionLabelsLoaded()) && + (!c_label_file.empty())) { + fastdeploy::jni::AssetsLoaderUtils::LoadDetectionLabels(c_label_file); + } + // WARN: need to release manually in Java ! + return reinterpret_cast(c_model_ptr); // native model context +} + +// 通过传入的模型指针在C++层进行预测 +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_predictNative( + JNIEnv *env, jclass clazz, jlong native_model_context, + jobject argb8888_bitmap, jboolean saved, jstring saved_image_path, + jfloat score_threshold, jboolean rendering) { + if (native_model_context == 0) { + return 0; + } + cv::Mat c_bgr; + auto t = fastdeploy::jni::GetCurrentTime(); + if (!fastdeploy::jni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { + return 0; + } + LOGD("Read from bitmap costs %f ms", fastdeploy::jni::GetElapsedTime(t)); + auto c_model_ptr = reinterpret_cast( + native_model_context); + auto c_result_ptr = new fastdeploy::vision::DetectionResult(); + t = fastdeploy::jni::GetCurrentTime(); + if (!c_model_ptr->Predict(&c_bgr, c_result_ptr)) { + delete c_result_ptr; + return 0; + } + LOGD("Predict from native costs %f ms", fastdeploy::jni::GetElapsedTime(t)); + if (c_model_ptr->EnabledRecordTimeOfRuntime()) { + auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); + LOGD("Avg runtime costs %f ms", info_of_runtime["avg_time"] * 1000.0f); + } + if (!c_result_ptr->boxes.empty() && rendering) { + t = fastdeploy::jni::GetCurrentTime(); + cv::Mat c_vis_im; + if (fastdeploy::jni::AssetsLoaderUtils::IsDetectionLabelsLoaded()) { + c_vis_im = fastdeploy::vision::VisDetection( + c_bgr, *(c_result_ptr), + fastdeploy::jni::AssetsLoaderUtils::GetDetectionLabels(), + score_threshold, 2, 1.0f); + } else { + c_vis_im = fastdeploy::vision::VisDetection(c_bgr, *(c_result_ptr), + score_threshold, 2, 1.0f); + } + LOGD("Visualize from native costs %f ms", + fastdeploy::jni::GetElapsedTime(t)); + // Rendering to bitmap + t = fastdeploy::jni::GetCurrentTime(); + if (!fastdeploy::jni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { + delete c_result_ptr; + return 0; + } + LOGD("Write to bitmap from native costs %f ms", + fastdeploy::jni::GetElapsedTime(t)); + std::string c_saved_image_path = + fastdeploy::jni::ConvertTo(env, saved_image_path); + if (!c_saved_image_path.empty() && saved) { + t = fastdeploy::jni::GetCurrentTime(); + cv::imwrite(c_saved_image_path, c_vis_im); + LOGD("Save image from native costs %f ms, path: %s", + fastdeploy::jni::GetElapsedTime(t), c_saved_image_path.c_str()); + } + } + // WARN: need to release it manually in Java ! + return reinterpret_cast(c_result_ptr); // native result context +} + +// 在C++层释放模型资源 +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_releaseNative( + JNIEnv *env, jclass clazz, jlong native_model_context) { + if (native_model_context == 0) { + return JNI_FALSE; + } + auto c_model_ptr = reinterpret_cast( + native_model_context); + if (c_model_ptr->EnabledRecordTimeOfRuntime()) { + auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); + LOGD("[End] Avg runtime costs %f ms", + info_of_runtime["avg_time"] * 1000.0f); + } + delete c_model_ptr; + LOGD("[End] Release PicoDet in native !"); + return JNI_TRUE; +} + +#ifdef __cplusplus +} +#endif +``` +## 编写CMakeLists.txt及配置build.gradle +实现好的JNI代码,需要被编译成so库,才能被Java调用,为实现该目的,需要在build.gradle中添加JNI项目支持,并编写对应的CMakeLists.txt。 +- build.gradle中配置NDK、CMake以及Android ABI +```java +android { + defaultConfig { + // 省略其他配置 ... + externalNativeBuild { + cmake { + arguments '-DANDROID_PLATFORM=android-21', '-DANDROID_STL=c++_shared', "-DANDROID_TOOLCHAIN=clang" + abiFilters 'armeabi-v7a', 'arm64-v8a' + cppFlags "-std=c++11" + } + } + } + // 省略其他配置 ... + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + version '3.10.2' + } + } + ndkVersion '20.1.5948944' +} +``` +- 编写CMakeLists.txt示例 +```cmake +cmake_minimum_required(VERSION 3.10.2) +project("fastdeploy_jni") + +set(FastDeploy_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../libs/fastdeploy-android-0.4.0-shared") + +find_package(FastDeploy REQUIRED) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${FastDeploy_INCLUDE_DIRS}) + +add_library( + fastdeploy_jni + SHARED + utils_jni.cc + bitmap_jni.cc + vision/results_jni.cc + vision/visualize_jni.cc + vision/detection/picodet_jni.cc + vision/classification/paddleclas_model_jni.cc) + +find_library(log-lib log) + +target_link_libraries( + # Specifies the target library. + fastdeploy_jni + jnigraphics + ${FASTDEPLOY_LIBS} + GLESv2 + EGL + ${log-lib} +) +``` +完整的工程示例,请参考 [android/app/src/main/cpp/CMakelists.txt](../../../examples/vision/detection/paddledetection/android/app/src/main/cpp/) 以及 [android/app/build.gradle](../../../examples/vision/detection/paddledetection/android/app/build.gradle). + +## 更多FastDeploy Android 使用案例请参考以下文档: +- [图像分类Android使用文档](../../../examples/vision/classification/paddleclas/android/README.md) +- [目标检测Android使用文档](../../../examples/vision/detection/paddledetection/android/README.md) diff --git a/examples/vision/classification/paddleclas/android/README.md b/examples/vision/classification/paddleclas/android/README.md index cd1579c846c..05164d8435a 100644 --- a/examples/vision/classification/paddleclas/android/README.md +++ b/examples/vision/classification/paddleclas/android/README.md @@ -152,4 +152,4 @@ set(FastDeploy_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../libs/fastdeploy-android ## 如何通过 JNI 在 Native 层接入 FastDeploy C++ API ? 如果您对如何通过JNI来接入FastDeploy C++ API感兴趣,可以参考以下内容: - [app/src/main/cpp 代码实现](./app/src/main/cpp/) -- [在 Android 中使用 FastDeploy C++ SDK](../../../../../docs/cn/faq/use_cpp_sdk_on_android.md) +- [在 Android 中通过 JNI 使用 FastDeploy C++ SDK](../../../../../docs/cn/faq/use_cpp_sdk_on_android.md) From 8f837d90dc5cd4d1dfe00dcb6ca51372786636fb Mon Sep 17 00:00:00 2001 From: DefTruth Date: Sat, 22 Oct 2022 21:55:10 +0800 Subject: [PATCH 10/10] [Doc] Update fastdeploy android jni usage docs --- docs/cn/faq/use_cpp_sdk_on_android.md | 86 ++++++++++----------------- 1 file changed, 30 insertions(+), 56 deletions(-) diff --git a/docs/cn/faq/use_cpp_sdk_on_android.md b/docs/cn/faq/use_cpp_sdk_on_android.md index dc08297dd83..bdea635b4b2 100644 --- a/docs/cn/faq/use_cpp_sdk_on_android.md +++ b/docs/cn/faq/use_cpp_sdk_on_android.md @@ -1,7 +1,16 @@ # 在 Android 中通过 JNI 使用 FastDeploy C++ SDK 本文档将以PicoDet为例,讲解如何通过JNI,将FastDeploy中的模型封装到Android中进行调用。阅读本文档,您至少需要了解C++、Java、JNI以及Android的基础知识。如果您主要关注如何在Java层如何调用FastDeploy的API,则可以不阅读本文档。 -## 新建PicoDet Java类及定义需要C++层实现的native API +## 目录 +- [新建Java类并定义native API](#Java) +- [Android Studio 生成JNI函数定义](#JNI) +- [在C++层实现JNI函数](#CPP) +- [编写CMakeLists.txt及配置build.gradle](#CMakeAndGradle) +- [更多FastDeploy Android 使用案例](#Examples) + +## 新建Java类并定义native API +
+ ```java public class PicoDet { protected long mNativeModelContext = 0; // Context from native. @@ -36,29 +45,32 @@ public class PicoDet { } ``` 这些被标记为native的接口是需要通过JNI的方式实现,并在Java层供PicoDet类调用。完整的PicoDet Java代码请参考 [PicoDet.java](../../../examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java) 。各个函数说明如下: -- bindNative: C++层初始化模型资源,如果成功初始化,则返回指向该模型的指针(long类型),否则返回0指针 -- predictNative: 通过已经初始化好的模型指针,在C++层执行预测代码,如果预测成功则返回指向预测结果的指针,否则返回0指针。注意,该结果指针在当次预测使用完之后需要释放,具体操作请参考 [PicoDet.java](../../../examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java) 中的predict函数。 -- releaseNative: 根据传入的模型指针,在C++层释放模型资源。 +- `bindNative`: C++层初始化模型资源,如果成功初始化,则返回指向该模型的指针(long类型),否则返回0指针 +- `predictNative`: 通过已经初始化好的模型指针,在C++层执行预测代码,如果预测成功则返回指向预测结果的指针,否则返回0指针。注意,该结果指针在当次预测使用完之后需要释放,具体操作请参考 [PicoDet.java](../../../examples/vision/detection/paddledetection/android/app/src/main/java/com/baidu/paddle/fastdeploy/vision/detection/PicoDet.java) 中的predict函数。 +- `releaseNative`: 根据传入的模型指针,在C++层释放模型资源。 -## Android Studio 生成JNI函数定义 -- Android Studio 生成 JNI 函数定义: 鼠标停留在Java中定义的native函数上,Android Studio 便会提示是否要创建JNI函数定义;这里,我们把JNI函数定义创建在一个事先创建好的c++文件`picodet_jni.cc`上; +## Android Studio 生成JNI函数定义 +
-使用Android Studio创建JNI函数定义: +Android Studio 生成 JNI 函数定义: 鼠标停留在Java中定义的native函数上,Android Studio 便会提示是否要创建JNI函数定义;这里,我们把JNI函数定义创建在一个事先创建好的c++文件`picodet_jni.cc`上; + +- 使用Android Studio创建JNI函数定义: ![](https://user-images.githubusercontent.com/31974251/197341065-cdf8f626-4bb1-4a57-8d7a-80b382fe994e.png) -将JNI函数定义创建在picodet_jni.cc上: +- 将JNI函数定义创建在picodet_jni.cc上: ![](https://user-images.githubusercontent.com/31974251/197341190-b887dec5-fa75-43c9-9ab3-7ead50c0eb45.png) -创建的JNI函数定义如下: +- 创建的JNI函数定义如下: ![](https://user-images.githubusercontent.com/31974251/197341274-e9671bac-9e77-4043-a870-9d5db914586b.png) 其他native函数对应的JNI函数定义的创建和此流程一样。 ## 在C++层实现JNI函数 +
+ 以下为PicoDet JNI层实现的示例,相关的辅助函数不在此处赘述,完整的C++代码请参考 [android/app/src/main/cpp](../../../examples/vision/detection/paddledetection/android/app/src/main/cpp/). ```C++ #include // NOLINT - #include "fastdeploy_jni.h" // NOLINT #ifdef __cplusplus @@ -115,11 +127,9 @@ Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_predictNative( return 0; } cv::Mat c_bgr; - auto t = fastdeploy::jni::GetCurrentTime(); if (!fastdeploy::jni::ARGB888Bitmap2BGR(env, argb8888_bitmap, &c_bgr)) { return 0; } - LOGD("Read from bitmap costs %f ms", fastdeploy::jni::GetElapsedTime(t)); auto c_model_ptr = reinterpret_cast( native_model_context); auto c_result_ptr = new fastdeploy::vision::DetectionResult(); @@ -128,43 +138,7 @@ Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_predictNative( delete c_result_ptr; return 0; } - LOGD("Predict from native costs %f ms", fastdeploy::jni::GetElapsedTime(t)); - if (c_model_ptr->EnabledRecordTimeOfRuntime()) { - auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); - LOGD("Avg runtime costs %f ms", info_of_runtime["avg_time"] * 1000.0f); - } - if (!c_result_ptr->boxes.empty() && rendering) { - t = fastdeploy::jni::GetCurrentTime(); - cv::Mat c_vis_im; - if (fastdeploy::jni::AssetsLoaderUtils::IsDetectionLabelsLoaded()) { - c_vis_im = fastdeploy::vision::VisDetection( - c_bgr, *(c_result_ptr), - fastdeploy::jni::AssetsLoaderUtils::GetDetectionLabels(), - score_threshold, 2, 1.0f); - } else { - c_vis_im = fastdeploy::vision::VisDetection(c_bgr, *(c_result_ptr), - score_threshold, 2, 1.0f); - } - LOGD("Visualize from native costs %f ms", - fastdeploy::jni::GetElapsedTime(t)); - // Rendering to bitmap - t = fastdeploy::jni::GetCurrentTime(); - if (!fastdeploy::jni::BGR2ARGB888Bitmap(env, argb8888_bitmap, c_vis_im)) { - delete c_result_ptr; - return 0; - } - LOGD("Write to bitmap from native costs %f ms", - fastdeploy::jni::GetElapsedTime(t)); - std::string c_saved_image_path = - fastdeploy::jni::ConvertTo(env, saved_image_path); - if (!c_saved_image_path.empty() && saved) { - t = fastdeploy::jni::GetCurrentTime(); - cv::imwrite(c_saved_image_path, c_vis_im); - LOGD("Save image from native costs %f ms, path: %s", - fastdeploy::jni::GetElapsedTime(t), c_saved_image_path.c_str()); - } - } - // WARN: need to release it manually in Java ! + // ... return reinterpret_cast(c_result_ptr); // native result context } @@ -177,13 +151,8 @@ Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_releaseNative( } auto c_model_ptr = reinterpret_cast( native_model_context); - if (c_model_ptr->EnabledRecordTimeOfRuntime()) { - auto info_of_runtime = c_model_ptr->PrintStatisInfoOfRuntime(); - LOGD("[End] Avg runtime costs %f ms", - info_of_runtime["avg_time"] * 1000.0f); - } + // ... delete c_model_ptr; - LOGD("[End] Release PicoDet in native !"); return JNI_TRUE; } @@ -192,6 +161,8 @@ Java_com_baidu_paddle_fastdeploy_vision_detection_PicoDet_releaseNative( #endif ``` ## 编写CMakeLists.txt及配置build.gradle +
+ 实现好的JNI代码,需要被编译成so库,才能被Java调用,为实现该目的,需要在build.gradle中添加JNI项目支持,并编写对应的CMakeLists.txt。 - build.gradle中配置NDK、CMake以及Android ABI ```java @@ -252,6 +223,9 @@ target_link_libraries( ``` 完整的工程示例,请参考 [android/app/src/main/cpp/CMakelists.txt](../../../examples/vision/detection/paddledetection/android/app/src/main/cpp/) 以及 [android/app/build.gradle](../../../examples/vision/detection/paddledetection/android/app/build.gradle). -## 更多FastDeploy Android 使用案例请参考以下文档: +## 更多FastDeploy Android 使用案例 +
+ +更多FastDeploy Android 使用案例请参考以下文档: - [图像分类Android使用文档](../../../examples/vision/classification/paddleclas/android/README.md) - [目标检测Android使用文档](../../../examples/vision/detection/paddledetection/android/README.md)