fix torch2onnx for mmdet3d (#2425)

* fix mmdet3d * fix * resolve comments * fix rtmdet ncnn * update docs
open-mmlab · Sep 14, 2023 · 71a1e6c · 71a1e6c
1 parent ec35b40
commit 71a1e6c
Show file tree

Hide file tree

Showing 12 changed files with 64 additions and 86 deletions.
diff --git a/.github/md-link-config.json b/.github/md-link-config.json
@@ -15,6 +15,9 @@
     },
     {
       "pattern": "^http://localhost"
+    },
+    {
+      "pattern": "^https://openmmlab.com"
     }
   ],
   "httpHeaders": [

diff --git a/.github/workflows/regression-test.yml b/.github/workflows/regression-test.yml
@@ -17,7 +17,7 @@ on:
         required: true
         description: 'Whether to start regression test on Linux x86_64'
         type: boolean
-        default: false
+        default: true
       test_windows:
         required: true
         description: 'Whether to start regression test on Windows'
@@ -52,7 +52,7 @@ on:
         required: false
         description: 'Do not change it unless you know what you are doing!'
         type: string
-        default: 'https://e2e1-14-136-99-158.ngrok-free.app'
+        default: 'https://4ec1-14-136-99-158.ngrok-free.app'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/configs/mmdet/detection/single-stage_ncnn_static-640x640.py b/configs/mmdet/detection/single-stage_ncnn_static-640x640.py
@@ -0,0 +1,4 @@
+_base_ = ['../_base_/base_static.py', '../../_base_/backends/ncnn.py']
+
+codebase_config = dict(model_type='ncnn_end2end')
+onnx_config = dict(output_names=['detection_output'], input_shape=[640, 640])
diff --git a/docs/en/04-supported-codebases/mmdet3d.md b/docs/en/04-supported-codebases/mmdet3d.md
@@ -12,49 +12,28 @@ ______________________________________________________________________
 
 ## Install mmdet3d
 
-These branches are required for mmdet3d deployment
-
-| codebase |  commit   |
-| :------: | :-------: |
-| mmdet3d  | v1.1.0rc1 |
-|   mmcv   | v2.0.0rc1 |
-|  mmdet   | v3.0.0rc1 |
-|  mmseg   | v1.0.0rc0 |
-
-First checkout and install mmcv/mmdet/mmseg/mmdet3d
-
-```bash
-python3 -m pip install openmim --user
-python3 -m mim install mmcv==2.0.0rc1 mmdet==3.0.0rc1 mmseg==1.0.0rc0 --user
-
-git clone https://github.com/open-mmlab/mmdetection3d --branch v1.1.0rc1
-cd mmdetection3d
-python3 -m pip install .
-cd -
-```
-
-After installation, `tools/check_env.py` should display mmdet3d version normally
+We could install mmdet3d through [mim](https://github.com/open-mmlab/mim).
+For other ways of installation, please refer to [here](https://mmdetection3d.readthedocs.io/en/latest/get_started.html#installation)
 
 ```bash
-python3 tools/check_env.py
-..
-11/11 13:56:19 - mmengine - INFO - **********Codebase information**********
-11/11 13:56:19 - mmengine - INFO - mmdet:       3.0.0rc1
-11/11 13:56:19 - mmengine - INFO - mmseg:       1.0.0rc0
-..
-11/11 13:56:19 - mmengine - INFO - mmdet3d:     1.1.0rc1
+python3 -m pip install -U openmim
+python3 -m mim install "mmdet3d>=1.1.0"
 ```
 
 ## Convert model
 
 For example, use `tools/deploy.py` to convert centerpoint to onnxruntime format
 
 ```bash
-export MODEL_CONFIG=/path/to/mmdetection3d/configs/centerpoint/centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py
+# cd to mmdeploy root directory
+# download config and model
+mim download mmdet3d --config centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d --dest .
+
+export MODEL_CONFIG=centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py
 
-export MODEL_PATH=https://download.openmmlab.com/mmdetection3d/v1.0.0_models/centerpoint/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20210816_064624-0f3299c0.pth
+export MODEL_PATH=centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20220811_031844-191a3822.pth
 
-export TEST_DATA=/path/to/mmdetection3d/tests/data/nuscenes/sweeps/LIDAR_TOP/n008-2018-09-18-12-07-26-0400__LIDAR_TOP__1537287083900561.pcd.bin
+export TEST_DATA=tests/data/n008-2018-08-01-15-16-36-0400__LIDAR_TOP__1533151612397179.pcd.bin
 
 python3 tools/deploy.py configs/mmdet3d/voxel-detection/voxel-detection_onnxruntime_dynamic.py $MODEL_CONFIG $MODEL_PATH $TEST_DATA --work-dir centerpoint
 ```
@@ -82,4 +61,4 @@ The caller needs to refer to the corresponding [python implementation](../../../
 |            [pointpillars](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py)            |   voxel detection   |  KITTI   |     ✔️      |    ✔️    |     ✔️     |
 |                   [smoke](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py)                    | monocular detection |  KITTI   |     ✔️      |    x     |     ✔️     |
 
-- Make sure trt >= 8.4 for some bug fixed, such as ScatterND, dynamic shape crash and so on.
+- Make sure trt >= 8.6 for some bug fixed, such as ScatterND, dynamic shape crash and so on.
diff --git a/docs/zh_cn/04-supported-codebases/mmdet3d.md b/docs/zh_cn/04-supported-codebases/mmdet3d.md
@@ -12,49 +12,28 @@ ______________________________________________________________________
 
 ## 安装 mmdet3d
 
-因为依赖的 codebase 不在 master 分支，所以要切到相应分支：
-
-| codebase |  commit   |
-| :------: | :-------: |
-| mmdet3d  | v1.1.0rc1 |
-|   mmcv   | v2.0.0rc1 |
-|  mmdet   | v3.0.0rc1 |
-|  mmseg   | v1.0.0rc0 |
-
-先安装前置依赖 mmcv/mmdet/mmseg，再安装 mmdet3d
-
-```bash
-python3 -m pip install openmim --user
-python3 -m mim install mmcv==2.0.0rc1 mmdet==3.0.0rc1 mmseg==1.0.0rc0 --user
-
-git clone https://github.com/open-mmlab/mmdetection3d --branch v1.1.0rc1
-cd mmdetection3d
-python3 -m pip install .
-cd -
-```
-
-成功后 `tools/check_env.py` 应能正常显示 mmdet3d 版本号。
+我们可以通过 [mim](https://github.com/open-mmlab/mim) 来安装 mmdet3d.
+更多安装方式可参考该[文档](https://mmdetection3d.readthedocs.io/en/latest/get_started.html#installation)
 
 ```bash
-python3 tools/check_env.py
-..
-11/11 13:56:19 - mmengine - INFO - **********Codebase information**********
-11/11 13:56:19 - mmengine - INFO - mmdet:       3.0.0rc1
-11/11 13:56:19 - mmengine - INFO - mmseg:       1.0.0rc0
-..
-11/11 13:56:19 - mmengine - INFO - mmdet3d:     1.1.0rc1
+python3 -m pip install -U openmim
+python3 -m mim install "mmdet3d>=1.1.0"
 ```
 
 ## 模型转换
 
 使用 `tools/deploy.py` 把 mmdet3d 转到相应后端，以 centerpoint onnxruntime 为例：
 
 ```bash
-export MODEL_CONFIG=/path/to/mmdetection3d/configs/centerpoint/centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py
+# 切换到 mmdeploy 根目录
+# 通过mim下载centerpoint模型
+mim download mmdet3d --config centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d --dest .
+
+export MODEL_CONFIG=centerpoint_pillar02_second_secfpn_head-circlenms_8xb4-cyclic-20e_nus-3d.py
 
-export MODEL_PATH=https://download.openmmlab.com/mmdetection3d/v1.0.0_models/centerpoint/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus/centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20210816_064624-0f3299c0.pth
+export MODEL_PATH=centerpoint_02pillar_second_secfpn_circlenms_4x8_cyclic_20e_nus_20220811_031844-191a3822.pth
 
-export TEST_DATA=/path/to/mmdetection3d/tests/data/nuscenes/sweeps/LIDAR_TOP/n008-2018-09-18-12-07-26-0400__LIDAR_TOP__1537287083900561.pcd.bin
+export TEST_DATA=tests/data/n008-2018-08-01-15-16-36-0400__LIDAR_TOP__1533151612397179.pcd.bin
 
 python3 tools/deploy.py configs/mmdet3d/voxel-detection/voxel-detection_onnxruntime_dynamic.py $MODEL_CONFIG $MODEL_PATH $TEST_DATA --work-dir centerpoint
 ```
@@ -80,4 +59,4 @@ ls -lah centerpoint
 |            [pointpillars](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/pointpillars/pointpillars_hv_secfpn_8xb6-160e_kitti-3d-3class.py)            |   voxel detection   |  KITTI   |     ✔️      |    ✔️    |     ✔️     |
 |                   [smoke](https://github.com/open-mmlab/mmdetection3d/blob/main/configs/smoke/smoke_dla34_dlaneck_gn-all_4xb8-6x_kitti-mono3d.py)                    | monocular detection |  KITTI   |     ✔️      |    x     |     ✔️     |
 
-- 考虑到 ScatterND、动态 shape 等已知问题，请确保 trt >= 8.4
+- 考虑到 ScatterND、动态 shape 等已知问题，请确保 trt >= 8.6
diff --git a/mmdeploy/codebase/mmdet/deploy/object_detection.py b/mmdeploy/codebase/mmdet/deploy/object_detection.py
@@ -70,8 +70,10 @@ def process_model_config(model_cfg: Config,
             if transform.type == 'Resize':
                 pipeline[i].keep_ratio = False
                 pipeline[i].scale = tuple(input_shape)
-            if transform.type in ('YOLOv5KeepRatioResize', 'LetterResize'):
+            elif transform.type in ('YOLOv5KeepRatioResize', 'LetterResize'):
                 pipeline[i].scale = tuple(input_shape)
+            elif transform.type == 'Pad' and 'size' in transform:
+                pipeline[i].size = tuple(input_shape)
 
     pipeline = [
         transform for transform in pipeline

diff --git a/mmdeploy/codebase/mmdet3d/deploy/mono_detection.py b/mmdeploy/codebase/mmdet3d/deploy/mono_detection.py
@@ -124,7 +124,8 @@ def create_input(
 
         if data_preprocessor is not None:
             collate_data = data_preprocessor(collate_data, False)
-            inputs = collate_data['inputs']
+            assert 'inputs' in collate_data
+            inputs = collate_data['inputs']['imgs']
         else:
             inputs = collate_data['inputs']
         return collate_data, inputs

diff --git a/mmdeploy/codebase/mmdet3d/models/base.py b/mmdeploy/codebase/mmdet3d/models/base.py
@@ -10,16 +10,18 @@
     'mmdet3d.models.detectors.Base3DDetector.forward'  # noqa: E501
 )
 def basedetector__forward(self,
-                          inputs: list,
+                          voxels: torch.Tensor,
+                          num_points: torch.Tensor,
+                          coors: torch.Tensor,
                           data_samples=None,
                           **kwargs) -> Tuple[List[torch.Tensor]]:
     """Extract features of images."""
 
     batch_inputs_dict = {
         'voxels': {
-            'voxels': inputs[0],
-            'num_points': inputs[1],
-            'coors': inputs[2]
+            'voxels': voxels,
+            'num_points': num_points,
+            'coors': coors
         }
     }
     return self._forward(batch_inputs_dict, data_samples, **kwargs)
diff --git a/mmdeploy/codebase/mmdet3d/models/mvx_two_stage.py b/mmdeploy/codebase/mmdet3d/models/mvx_two_stage.py
@@ -46,11 +46,15 @@ def mvxtwostagedetector__extract_feat(self, batch_inputs_dict: dict) -> tuple:
 
 @FUNCTION_REWRITER.register_rewriter(
     'mmdet3d.models.detectors.mvx_two_stage.MVXTwoStageDetector.forward')
-def mvxtwostagedetector__forward(self, inputs: list, **kwargs):
+def mvxtwostagedetector__forward(self, voxels: torch.Tensor,
+                                 num_points: torch.Tensor, coors: torch.Tensor,
+                                 **kwargs):
     """Rewrite this func to remove voxelize op.
 
     Args:
-        inputs (list): input list comprises voxels, num_points and coors
+        voxels (Tensor): input voxels
+        num_points (Tensor): input num_points
+        coors (Tensor): input coors
 
     Returns:
         tuple: A tuple of classification scores, bbox and direction
@@ -70,9 +74,9 @@ def mvxtwostagedetector__forward(self, inputs: list, **kwargs):
     deploy_cfg = ctx.cfg
     batch_inputs_dict = {
         'voxels': {
-            'voxels': inputs[0],
-            'num_points': inputs[1],
-            'coors': inputs[2]
+            'voxels': voxels,
+            'num_points': num_points,
+            'coors': coors
         }
     }
 

diff --git a/mmdeploy/codebase/mmdet3d/models/single_stage_mono3d.py b/mmdeploy/codebase/mmdet3d/models/single_stage_mono3d.py
@@ -1,19 +1,22 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+from torch import Tensor
+
 from mmdeploy.core import FUNCTION_REWRITER
 
 
 @FUNCTION_REWRITER.register_rewriter(
     'mmdet3d.models.detectors.single_stage_mono3d.'
     'SingleStageMono3DDetector.forward')
-def singlestagemono3ddetector__forward(self, inputs: list, **kwargs):
-    """Rewrite this func to r.
+def singlestagemono3ddetector__forward(self, inputs: Tensor, **kwargs):
+    """Rewrite to support feed inputs of Tensor type.
 
     Args:
-        inputs (dict): Input dict comprises `imgs`
+        inputs (Tensor): Input image
 
     Returns:
         list: two torch.Tensor
     """
-    x = self.extract_feat(inputs)
+
+    x = self.extract_feat({'imgs': inputs})
     results = self.bbox_head.forward(x)
     return results[0], results[1]
diff --git a/tests/regression/mmdet.yml b/tests/regression/mmdet.yml
@@ -348,10 +348,11 @@ models:
       - configs/rtmdet/rtmdet_s_8xb32-300e_coco.py
     pipelines:
       - *pipeline_ort_dynamic_fp32
-      - deploy_config: configs/mmdet/detection/detection_tensorrt_dynamic-64x64-800x800.py
+      - deploy_config: configs/mmdet/detection/detection_tensorrt_static-640x640.py
         convert_image: *convert_image
         backend_test: *default_backend_test
-        sdk_config: *sdk_dynamic
+      - deploy_config: configs/mmdet/detection/single-stage_ncnn_static-640x640.py
+        convert_image: *convert_image
 
   - name: SOLO
     metafile: configs/solo/metafile.yml

diff --git a/tests/test_codebase/test_mmdet3d/test_mmdet3d_models.py b/tests/test_codebase/test_mmdet3d/test_mmdet3d_models.py
@@ -157,7 +157,7 @@ def test_pointpillars(backend_type: Backend):
             cfg=deploy_cfg,
             backend=deploy_cfg.backend_config.type,
             opset=deploy_cfg.onnx_config.opset_version):
-        outputs = model.forward(data)
+        outputs = model.forward(*data)
         assert len(outputs) == 3