diff --git a/configs/mmcls/mmcls_regression_test.yaml b/configs/mmcls/mmcls_regression_test.yaml new file mode 100644 index 000000000..e47185ecc --- /dev/null +++ b/configs/mmcls/mmcls_regression_test.yaml @@ -0,0 +1,190 @@ +globals: + codebase_name: mmcls + codebase_dir: ../mmclassification + checkpoint_force_download: False + checkpoint_dir: ../mmdeploy_checkpoints + images: + img_snake: &img_snake ../mmclassification/demo/demo.JPEG + img_bird: &img_bird ../mmclassification/demo/bird.JPEG + img_cat_dog: &img_cat_dog ../mmclassification/demo/cat-dog.png + img_dog: &img_dog ../mmclassification/demo/dog.jpg + img_color_cat: &img_color_cat ../mmclassification/tests/data/color.jpg + img_gray_cat: &img_gray_cat ../mmclassification/tests/data/gray.jpg + + metric_info: &metric_info + Top 1 Accuracy: # named after metafile.Results.Metrics + eval_name: accuracy # test.py --metrics args + metric_key: accuracy_top-1 # eval Dict key name + tolerance: 1 # metric ±n% + task_name: Image Classification # metafile.Results.Task + dataset: ImageNet-1k # metafile.Results.Dataset + Top 5 Accuracy: + eval_name: accuracy + metric_key: accuracy_top-5 + tolerance: 1 # metric ±n% + task_name: Image Classification + dataset: ImageNet-1k + convert_image: &convert_image + input_img: *img_snake + test_img: *img_color_cat + backend_test: &default_backend_test True + sdk: + sdk_dynamic: &sdk_dynamic configs/mmcls/classification_sdk_dynamic.py + + +onnxruntime: + pipeline_ort_static_fp32: &pipeline_ort_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmcls/classification_onnxruntime_static.py + + pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmcls/classification_onnxruntime_dynamic.py + + +tensorrt: + pipeline_trt_static_fp32: &pipeline_trt_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + deploy_config: configs/mmcls/classification_tensorrt_static-224x224.py + + pipeline_trt_static_fp16: &pipeline_trt_static_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + deploy_config: configs/mmcls/classification_tensorrt-fp16_static-224x224.py + + pipeline_trt_static_int8: &pipeline_trt_static_int8 + convert_image: *convert_image + backend_test: *default_backend_test + deploy_config: configs/mmcls/classification_tensorrt-int8_static-224x224.py + + pipeline_trt_dynamic_fp32: &pipeline_trt_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmcls/classification_tensorrt_dynamic-224x224-224x224.py + + pipeline_trt_dynamic_fp16: &pipeline_trt_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmcls/classification_tensorrt-fp16_dynamic-224x224-224x224.py + + pipeline_trt_dynamic_int8: &pipeline_trt_dynamic_int8 + convert_image: *convert_image + calib_dataset_cfg: + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmcls/classification_tensorrt-int8_dynamic-224x224-224x224.py + + +openvino: + pipeline_openvino_dynamic_fp32: &pipeline_openvino_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmcls/classification_openvino_dynamic-224x224.py + + +ncnn: + pipeline_ncnn_static_fp32: &pipeline_ncnn_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmcls/classification_ncnn_static.py + + pipeline_ncnn_dynamic_fp32: &pipeline_ncnn_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmcls/classification_ncnn_dynamic.py + + +pplnn: + pipeline_pplnn_dynamic_fp32: &pipeline_pplnn_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmcls/classification_pplnn_dynamic-224x224.py + + +torchscript: + pipeline_ts_fp32: &pipeline_ts_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmcls/classification_torchscript.py + + +models: + - name: ResNet + metafile: configs/resnet/metafile.yml + model_configs: + - configs/resnet/resnet18_8xb32_in1k.py # TODO Not benchmark config + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 +# - *pipeline_trt_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 +# - *pipeline_trt_dynamic_int8 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: ResNeXt + metafile: configs/resnext/metafile.yml + model_configs: + - configs/resnext/resnext50-32x4d_8xb32_in1k.py # TODO Not benchmark config + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: SE-ResNet + metafile: configs/seresnet/metafile.yml + model_configs: + - configs/seresnet/seresnet50_8xb32_in1k.py # TODO Not benchmark config + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: MobileNetV2 + metafile: configs/mobilenet_v2/metafile.yml + model_configs: + - configs/mobilenet_v2/mobilenet-v2_8xb32_in1k.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: ShuffleNetV1 + metafile: configs/shufflenet_v1/metafile.yml + model_configs: + - configs/shufflenet_v1/shufflenet-v1-1x_16xb64_in1k.py + pipelines: + - *pipeline_ts_fp32 +# - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_ncnn_static_fp32 +# - *pipeline_pplnn_dynamic_fp32 +# - *pipeline_openvino_dynamic_fp32 + + - name: ShuffleNetV2 + metafile: configs/shufflenet_v2/metafile.yml + model_configs: + - configs/shufflenet_v2/shufflenet-v2-1x_16xb64_in1k.py + pipelines: + - *pipeline_ts_fp32 +# - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_ncnn_static_fp32 +# - *pipeline_pplnn_dynamic_fp32 +# - *pipeline_openvino_dynamic_fp32 diff --git a/configs/mmdet/detection/detection_openvino_dynamic-300x300.py b/configs/mmdet/detection/detection_openvino_dynamic-300x300.py index 1df7d1211..ee879c361 100644 --- a/configs/mmdet/detection/detection_openvino_dynamic-300x300.py +++ b/configs/mmdet/detection/detection_openvino_dynamic-300x300.py @@ -1 +1 @@ -_base_ = ['../_base_/base_openvino_dynamic.py'] +_base_ = ['../_base_/base_openvino_dynamic-300x300.py'] diff --git a/configs/mmdet/mmdet_regression_test.yaml b/configs/mmdet/mmdet_regression_test.yaml new file mode 100644 index 000000000..22a8688d3 --- /dev/null +++ b/configs/mmdet/mmdet_regression_test.yaml @@ -0,0 +1,304 @@ +globals: + codebase_name: mmdet + codebase_dir: ../mmdetection + checkpoint_force_download: False + checkpoint_dir: ../mmdeploy_checkpoints + images: + input_img: &input_img ../mmdetection/demo/demo.jpg + test_img: &test_img ./tests/data/tiger.jpeg + img_blank: &img_blank + metric_info: &metric_info + box AP: # named after metafile.Results.Metrics + eval_name: bbox # test.py --metrics args + metric_key: bbox_mAP # eval OrderedDict key name + tolerance: 0.2 # metric ±n% + task_name: Object Detection # metafile.Results.Task + dataset: COCO # metafile.Results.Dataset + mask AP: + eval_name: segm + metric_key: segm_mAP + tolerance: 1 # metric ±n% + task_name: Instance Segmentation + dataset: COCO + PQ: + eval_name: proposal + metric_key: '?' + tolerance: 0.1 # metric ±n% + task_name: Panoptic Segmentation + dataset: COCO + convert_image: &convert_image + input_img: *input_img + test_img: *test_img + backend_test: &default_backend_test True + sdk: + sdk_static: &sdk_static configs/mmdet/detection/detection_sdk_static.py + sdk_dynamic: &sdk_dynamic configs/mmdet/detection/detection_sdk_dynamic.py +# sdk_seg_static: &sdk_seg_static configs/mmdet/instance-seg/instance-seg_sdk_static.py + sdk_seg_dynamic: &sdk_seg_dynamic configs/mmdet/instance-seg/instance-seg_sdk_dynamic.py + +onnxruntime: + pipeline_ort_static_fp32: &pipeline_ort_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmdet/detection/detection_onnxruntime_static.py + + pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmdet/detection/detection_onnxruntime_dynamic.py + + pipeline_seg_ort_static_fp32: &pipeline_seg_ort_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test +# sdk_config: *sdk_seg_static + deploy_config: configs/mmdet/instance-seg/instance-seg_onnxruntime_static.py + + pipeline_seg_ort_dynamic_fp32: &pipeline_seg_ort_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_onnxruntime_dynamic.py + +tensorrt: + pipeline_trt_static_fp32: &pipeline_trt_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/detection_tensorrt_static-800x1344.py + + pipeline_trt_static_fp16: &pipeline_trt_static_fp16 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/detection_tensorrt-fp16_static-800x1344.py + + pipeline_trt_static_int8: &pipeline_trt_static_int8 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/detection_tensorrt-int8_static-800x1344.py + + pipeline_trt_dynamic_fp32: &pipeline_trt_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmdet/detection/detection_tensorrt_dynamic-320x320-1344x1344.py + + pipeline_trt_dynamic_fp16: &pipeline_trt_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmdet/detection/detection_tensorrt-fp16_dynamic-320x320-1344x1344.py + + pipeline_trt_dynamic_int8: &pipeline_trt_dynamic_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmdet/detection/detection_tensorrt-int8_dynamic-320x320-1344x1344.py + + # ============= seg ================ + pipeline_seg_trt_static_fp32: &pipeline_seg_trt_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_tensorrt_static-800x1344.py + + pipeline_seg_trt_static_fp16: &pipeline_seg_trt_static_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_tensorrt-fp16_static-800x1344.py + + pipeline_seg_trt_static_int8: &pipeline_seg_trt_static_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_tensorrt-int8_static-800x1344.py + + pipeline_seg_trt_dynamic_fp32: &pipeline_seg_trt_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test +# sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_tensorrt_dynamic-320x320-1344x1344.py + + pipeline_seg_trt_dynamic_fp16: &pipeline_seg_trt_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_tensorrt-fp16_dynamic-320x320-1344x1344.py + + pipeline_seg_trt_dynamic_int8: &pipeline_seg_trt_dynamic_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_seg_dynamic + deploy_config: configs/mmdet/instance-seg/instance-seg_tensorrt-int8_dynamic-320x320-1344x1344.py + +openvino: + pipeline_openvino_dynamic_fp32: &pipeline_openvino_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/detection_openvino_dynamic-800x1344.py + + # ============= seg ================ + pipeline_seg_openvino_dynamic_fp32: &pipeline_seg_openvino_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/instance-seg/instance-seg_openvino_dynamic-800x1344.py + +ncnn: + pipeline_ncnn_static_fp32: &pipeline_ncnn_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/single-stage_ncnn_static-800x1344.py + + pipeline_ncnn_dynamic_fp32: &pipeline_ncnn_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/single-stage_ncnn_dynamic.py + +pplnn: + pipeline_pplnn_dynamic_fp32: &pipeline_pplnn_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + deploy_config: configs/mmdet/detection/detection_pplnn_dynamic-800x1344.py + + # ============= seg ================ + pipeline_seg_pplnn_dynamic_fp32: &pipeline_seg_pplnn_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + deploy_config: configs/mmdet/instance-seg/instance-seg_pplnn_dynamic-800x1344.py + +torchscript: + pipeline_ts_fp32: &pipeline_ts_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/detection/detection_torchscript.py + + # ============= seg ================ + pipeline_seg_ts_fp32: &pipeline_seg_ts_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmdet/instance-seg/instance-seg_torchscript.py + +models: + - name: YOLOV3 + metafile: configs/yolo/metafile.yml + model_configs: + - configs/yolo/yolov3_d53_320_273e_coco.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 +# - *pipeline_trt_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 +# - *pipeline_trt_dynamic_int8 + - *pipeline_ncnn_static_fp32 +# - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: SSD + metafile: configs/ssd/metafile.yml + model_configs: + - configs/ssd/ssd300_coco.py + pipelines: # special cases + - *pipeline_ts_fp32 + - *pipeline_ort_static_fp32 + - deploy_config: configs/mmdet/detection/detection_tensorrt-fp16_dynamic-300x300-512x512.py + convert_image: *convert_image + - deploy_config: configs/mmdet/detection/single-stage_ncnn_static-300x300.py + convert_image: *convert_image +# - *pipeline_openvino_static_fp32 + + - name: RetinaNet + metafile: configs/retinanet/metafile.yml + model_configs: + - configs/retinanet/retinanet_r50_fpn_1x_coco.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: FCOS + metafile: configs/fcos/metafile.yml + model_configs: + - configs/fcos/fcos_r50_caffe_fpn_gn-head_1x_coco.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: FSAF + metafile: configs/fsaf/metafile.yml + model_configs: + - configs/fsaf/fsaf_r50_fpn_1x_coco.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: YOLOX + metafile: configs/yolox/metafile.yml + model_configs: + - configs/yolox/yolox_s_8x8_300e_coco.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: Faster R-CNN + metafile: configs/faster_rcnn/metafile.yml + model_configs: + - configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: ATSS + metafile: configs/atss/metafile.yml + model_configs: + - configs/atss/atss_r50_fpn_1x_coco.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_openvino_dynamic_fp32 + + - name: Cascade R-CNN + metafile: configs/cascade_rcnn/metafile.yml + model_configs: + - configs/cascade_rcnn/cascade_rcnn_r50_caffe_fpn_1x_coco.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: GFL + metafile: configs/gfl/metafile.yml + model_configs: + - configs/gfl/gfl_r50_fpn_1x_coco.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_openvino_dynamic_fp32 + + - name: Mask R-CNN + metafile: configs/mask_rcnn/metafile.yml + model_configs: + - configs/mask_rcnn/mask_rcnn_r50_fpn_1x_coco.py + pipelines: + - *pipeline_seg_ts_fp32 + - *pipeline_seg_ort_dynamic_fp32 + - *pipeline_seg_trt_dynamic_fp32 + - *pipeline_seg_openvino_dynamic_fp32 diff --git a/configs/mmedit/mmedit_regression_test.yaml b/configs/mmedit/mmedit_regression_test.yaml new file mode 100644 index 000000000..b50d4355b --- /dev/null +++ b/configs/mmedit/mmedit_regression_test.yaml @@ -0,0 +1,163 @@ +globals: + codebase_name: mmedit + codebase_dir: ../mmediting + checkpoint_force_download: False + checkpoint_dir: ../mmdeploy_checkpoints + images: + img_face: &img_face ../mmediting/tests/data/face/000001.png + img_bg: &img_bg ../mmediting/tests/data/bg/GT26r.jpg + metric_info: &metric_info + PSNR: # named after metafile.Results.Metrics + eval_name: PSNR # test.py --metrics args + metric_key: Eval-PSNR # eval log key name + tolerance: 4 # metric ±n% + task_name: Restorers # metafile.Results.Task + dataset: Set5 # metafile.Results.Dataset + SSIM: + eval_name: SSIM + metric_key: Eval-SSIM + tolerance: 0.02 # metric ±n + task_name: Restorers + dataset: Set5 + convert_image: &convert_image + input_img: *img_face + test_img: *img_bg + backend_test: &default_backend_test True + sdk: + sdk_dynamic: &sdk_dynamic configs/mmedit/super-resolution/super-resolution_sdk_dynamic.py + +onnxruntime: + pipeline_ort_static_fp32: &pipeline_ort_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_onnxruntime_static.py + + pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32 + convert_image: *convert_image + deploy_config: configs/mmedit/super-resolution/super-resolution_onnxruntime_dynamic.py + +tensorrt: + pipeline_trt_static_fp32: &pipeline_trt_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_tensorrt_static-256x256.py + + pipeline_trt_static_fp16: &pipeline_trt_static_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_tensorrt-fp16_static-256x256.py + + pipeline_trt_static_int8: &pipeline_trt_static_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_tensorrt-int8_static-256x256.py + + pipeline_trt_dynamic_fp32: &pipeline_trt_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_tensorrt_dynamic-32x32-512x512.py + + pipeline_trt_dynamic_fp16: &pipeline_trt_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_tensorrt-fp16_dynamic-32x32-512x512.py + + pipeline_trt_dynamic_int8: &pipeline_trt_dynamic_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmedit/super-resolution/super-resolution_tensorrt-int8_dynamic-32x32-512x512.py + +openvino: + pipeline_openvino_dynamic_fp32: &pipeline_openvino_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmedit/super-resolution/super-resolution_openvino_dynamic-256x256.py + +ncnn: + pipeline_ncnn_dynamic_fp32: &pipeline_ncnn_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmedit/super-resolution/super-resolution_ncnn_dynamic.py + +pplnn: + pipeline_pplnn_dynamic_fp32: &pipeline_pplnn_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmedit/super-resolution/super-resolution_pplnn_dynamic-32x32.py + +torchscript: + pipeline_ts_fp32: &pipeline_ts_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmedit/super-resolution/super-resolution_torchscript.py + +models: + - name: SRCNN + metafile: configs/restorers/srcnn/metafile.yml + model_configs: + - configs/restorers/srcnn/srcnn_x4k915_g1_1000k_div2k.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 +# - *pipeline_trt_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 +# - *pipeline_trt_dynamic_int8 + - *pipeline_ncnn_dynamic_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: ESRGAN + metafile: configs/restorers/esrgan/metafile.yml + model_configs: + - configs/restorers/esrgan/esrgan_x4c64b23g32_g1_400k_div2k.py + - configs/restorers/esrgan/esrgan_psnr_x4c64b23g32_g1_1000k_div2k.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_dynamic_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: SRGAN + metafile: configs/restorers/srresnet_srgan/metafile.yml + model_configs: + - configs/restorers/srresnet_srgan/srgan_x4c64b16_g1_1000k_div2k.py + - configs/restorers/srresnet_srgan/msrresnet_x4c64b16_g1_1000k_div2k.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_dynamic_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: Real-ESRNet + metafile: configs/restorers/real_esrgan/metafile.yml + model_configs: + - configs/restorers/real_esrgan/realesrnet_c64b23g32_12x4_lr2e-4_1000k_df2k_ost.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_dynamic_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: EDSR + metafile: configs/restorers/edsr/metafile.yml + model_configs: + - configs/restorers/edsr/edsr_x4c64b16_g1_300k_div2k.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 diff --git a/configs/mmocr/mmocr_regression_test.yaml b/configs/mmocr/mmocr_regression_test.yaml new file mode 100644 index 000000000..f0baf4044 --- /dev/null +++ b/configs/mmocr/mmocr_regression_test.yaml @@ -0,0 +1,207 @@ +globals: + codebase_name: mmocr + codebase_dir: ../mmocr + checkpoint_force_download: False + checkpoint_dir: ../mmdeploy_checkpoints + images: + img_densetext_det: &img_densetext_det ../mmocr/demo/demo_densetext_det.jpg + img_demo_text_det: &img_demo_text_det ../mmocr/demo/demo_text_det.jpg + img_demo_text_ocr: &img_demo_text_ocr ../mmocr/demo/demo_text_ocr.jpg + img_demo_text_recog: &img_demo_text_recog ../mmocr/demo/demo_text_recog.jpg + metric_info: &metric_info + hmean-iou: # named after metafile.Results.Metrics + eval_name: hmean-iou # test.py --metrics args + metric_key: 0_hmean-iou:hmean # eval key name + tolerance: 0.15 # metric ±n% + task_name: Text Detection # metafile.Results.Task + dataset: ICDAR2015 # metafile.Results.Dataset + word_acc: + eval_name: acc + metric_key: 0_word_acc_ignore_case + tolerance: 0.05 # metric ±n% + task_name: Text Recognition + dataset: IIIT5K + convert_image_det: &convert_image_det + input_img: *img_densetext_det + test_img: *img_demo_text_det + convert_image_rec: &convert_image_rec + input_img: *img_demo_text_recog + test_img: *img_demo_text_recog + backend_test: &default_backend_test True + sdk: + sdk_detection_dynamic: &sdk_detection_dynamic configs/mmocr/text-detection/text-detection_sdk_dynamic.py + sdk_recognition_dynamic: &sdk_recognition_dynamic configs/mmocr/text-recognition/text-recognition_sdk_dynamic.py + +onnxruntime: + # ======= detection ======= + pipeline_ort_detection_static_fp32: &pipeline_ort_detection_static_fp32 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_onnxruntime_static.py + + pipeline_ort_detection_dynamic_fp32: &pipeline_ort_detection_dynamic_fp32 + convert_image: *convert_image_det + deploy_config: configs/mmocr/text-detection/text-detection_onnxruntime_dynamic.py + + # ======= recognition ======= + pipeline_ort_recognition_static_fp32: &pipeline_ort_recognition_static_fp32 + convert_image: *convert_image_rec + deploy_config: configs/mmocr/text-recognition/text-recognition_onnxruntime_static.py + + pipeline_ort_recognition_dynamic_fp32: &pipeline_ort_recognition_dynamic_fp32 + convert_image: *convert_image_rec + deploy_config: configs/mmocr/text-recognition/text-recognition_onnxruntime_dynamic.py + +tensorrt: + # ======= detection ======= + pipeline_trt_detection_static_fp32: &pipeline_trt_detection_static_fp32 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt_static-512x512.py + + pipeline_trt_detection_static_fp16: &pipeline_trt_detection_static_fp16 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt-fp16_static-512x512.py + + pipeline_trt_detection_static_int8: &pipeline_trt_detection_static_int8 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt-int8_static-512x512.py + + pipeline_trt_detection_dynamic_fp32: &pipeline_trt_detection_dynamic_fp32 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt_dynamic-320x320-1024x1824.py + + pipeline_trt_detection_dynamic_fp16: &pipeline_trt_detection_dynamic_fp16 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt-fp16_dynamic-320x320-1024x1824.py + + pipeline_trt_detection_dynamic_int8: &pipeline_trt_detection_dynamic_int8 + convert_image: *convert_image_det + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt-int8_dynamic-320x320-1024x1824.py + + # ======= recognition ======= + pipeline_trt_recognition_static_fp32: &pipeline_trt_recognition_static_fp32 + convert_image: *convert_image_rec + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt_static-1x32x32.py + + pipeline_trt_recognition_static_fp16: &pipeline_trt_recognition_static_fp16 + convert_image: *convert_image_rec + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt-fp16_static-1x32x32.py + + pipeline_trt_recognition_static_int8: &pipeline_trt_recognition_static_int8 + convert_image: *convert_image_rec + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt-int8_static-1x32x32.py + + pipeline_trt_recognition_dynamic_fp32: &pipeline_trt_recognition_dynamic_fp32 + convert_image: *convert_image_rec + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt_dynamic-1x32x32-1x32x640.py + + pipeline_trt_recognition_dynamic_fp16: &pipeline_trt_recognition_dynamic_fp16 + convert_image: *convert_image_rec + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt-fp16_dynamic-1x32x32-1x32x640.py + + pipeline_trt_recognition_dynamic_int8: &pipeline_trt_recognition_dynamic_int8 + convert_image: *convert_image_rec + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt-int8_dynamic-1x32x32-1x32x640.py + +openvino: + pipeline_openvino_detection_dynamic_fp32: &pipeline_openvino_detection_dynamic_fp32 + convert_image: *convert_image_det + backend_test: *default_backend_test + deploy_config: configs/mmocr/text-detection/text-detection_openvino_dynamic-640x640.py + +# pipeline_openvino_recognition_dynamic_fp32: &pipeline_openvino_recognition_dynamic_fp32 +# convert_image: *convert_image_rec +# backend_test: *default_backend_test +# deploy_config: + +ncnn: + pipeline_ncnn_detection_static_fp32: &pipeline_ncnn_detection_static_fp32 + convert_image: *convert_image_det + backend_test: False + deploy_config: configs/mmocr/text-detection/text-detection_ncnn_static.py + + pipeline_ncnn_recognition_static_fp32: &pipeline_ncnn_recognition_static_fp32 + convert_image: *convert_image_rec + backend_test: False + deploy_config: configs/mmocr/text-recognition/text-recognition_ncnn_static.py + +pplnn: + pipeline_pplnn_detection_dynamic_fp32: &pipeline_pplnn_detection_dynamic_fp32 + convert_image: *convert_image_det + backend_test: False + deploy_config: configs/mmocr/text-detection/text-detection_pplnn_dynamic-640x640.py + + pipeline_pplnn_recognition_dynamic_fp32: &pipeline_pplnn_recognition_dynamic_fp32 + convert_image: *convert_image_rec + backend_test: False + deploy_config: configs/mmocr/text-recognition/text-recognition_pplnn_dynamic-1x32x32.py + +torchscript: + pipeline_ts_detection_fp32: &pipeline_ts_detection_fp32 + convert_image: *convert_image_det + backend_test: False + deploy_config: configs/mmocr/text-detection/text-detection_torchscript.py + + pipeline_ts_recognition_fp32: &pipeline_ts_recognition_fp32 + convert_image: *convert_image_rec + backend_test: False + deploy_config: configs/mmocr/text-recognition/text-recognition_torchscript.py + + +models: + - name: DBNet + metafile: configs/textdet/dbnet/metafile.yml + model_configs: + - configs/textdet/dbnet/dbnet_r18_fpnc_1200e_icdar2015.py + pipelines: + - *pipeline_ts_detection_fp32 + - *pipeline_ort_detection_dynamic_fp32 +# - *pipeline_trt_detection_dynamic_fp32 + - *pipeline_trt_detection_dynamic_fp16 +# - *pipeline_trt_detection_dynamic_int8 + - *pipeline_ncnn_detection_static_fp32 + - *pipeline_pplnn_detection_dynamic_fp32 + - *pipeline_openvino_detection_dynamic_fp32 + + - name: CRNN + metafile: configs/textrecog/crnn/metafile.yml + model_configs: + - configs/textrecog/crnn/crnn_academic_dataset.py + pipelines: + - *pipeline_ts_recognition_fp32 + - *pipeline_ort_recognition_dynamic_fp32 + - *pipeline_trt_recognition_dynamic_fp16 + - *pipeline_ncnn_recognition_static_fp32 + - *pipeline_pplnn_recognition_dynamic_fp32 + + - name: SAR + metafile: configs/textrecog/sar/metafile.yml + model_configs: + - configs/textrecog/sar/sar_r31_parallel_decoder_academic.py + pipelines: + - *pipeline_ort_recognition_dynamic_fp32 diff --git a/configs/mmpose/mmpose_regression_test.yaml b/configs/mmpose/mmpose_regression_test.yaml new file mode 100644 index 000000000..9b8223920 --- /dev/null +++ b/configs/mmpose/mmpose_regression_test.yaml @@ -0,0 +1,104 @@ +globals: + codebase_name: mmpose + codebase_dir: ../mmpose + checkpoint_force_download: False + checkpoint_dir: ../mmdeploy_checkpoints + images: + img_human_pose: &img_human_pose ../mmpose/tests/data/coco/000000000785.jpg + img_human_pose_256x192: &img_human_pose_256x192 ./demo/resources/human-pose.jpg + img_blank: &img_blank + metric_info: &metric_info + AP: # named after metafile.Results.Metrics + eval_name: mAP # test.py --metrics args + metric_key: AP # eval key name + tolerance: 0.10 # metric ±n + task_name: Body 2D Keypoint # metafile.Results.Task + dataset: COCO # metafile.Results.Dataset + AR: + eval_name: mAP + metric_key: AR + tolerance: 0.08 # metric ±n + task_name: Body 2D Keypoint + dataset: COCO + convert_image: &convert_image + input_img: *img_human_pose + test_img: *img_human_pose_256x192 + backend_test: &default_backend_test True + sdk: + sdk_static: &sdk_static configs/mmpose/pose-detection_sdk_static.py + +onnxruntime: + pipeline_ort_static_fp32: &pipeline_ort_static_fp32 + convert_image: *convert_image + deploy_config: configs/mmpose/pose-detection_onnxruntime_static.py + +tensorrt: + pipeline_trt_static_fp32: &pipeline_trt_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_static + deploy_config: configs/mmpose/pose-detection_tensorrt_static-256x192.py + pipeline_trt_static_fp16: &pipeline_trt_static_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_static + deploy_config: configs/mmpose/pose-detection_tensorrt-fp16_static-256x192.py + pipeline_trt_static_int8: &pipeline_trt_static_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_static + deploy_config: configs/mmpose/pose-detection_tensorrt-int8_static-256x192.py + +openvino: + pipeline_openvino_static_fp32: &pipeline_openvino_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + deploy_config: configs/mmpose/pose-detection_openvino_static-256x192.py + +ncnn: + pipeline_ncnn_static_fp32: &pipeline_ncnn_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmpose/pose-detection_ncnn_static-256x192.py + +pplnn: + pipeline_pplnn_static_fp32: &pipeline_pplnn_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmpose/pose-detection_pplnn_static-256x192.py + +torchscript: + pipeline_ts_static_fp32: &pipeline_ts_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmpose/pose-detection_torchscript.py + +models: + - name: HRNET + metafile: configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/coco/hrnet_coco.yml + model_configs: + - configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/coco/hrnet_w48_coco_256x192.py + pipelines: + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_static_fp32 + + - name: LiteHRNet + metafile: configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/coco/litehrnet_coco.yml + model_configs: + - configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/coco/litehrnet_30_coco_256x192.py + pipelines: + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp32 + - *pipeline_openvino_static_fp32 + + - name: MSPN + metafile: configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/coco/mspn_coco.yml + model_configs: + - configs/body/2d_kpt_sview_rgb_img/topdown_heatmap/coco/4xmspn50_coco_256x192.py + pipelines: + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_static_fp32 diff --git a/configs/mmpose/pose-detection_torchscript.py b/configs/mmpose/pose-detection_torchscript.py new file mode 100644 index 000000000..ef813893a --- /dev/null +++ b/configs/mmpose/pose-detection_torchscript.py @@ -0,0 +1,6 @@ +_base_ = [ + '../_base_/torchscript_config.py', '../_base_/backends/torchscript.py' +] + +codebase_config = dict(type='mmpose', task='PoseDetection') +ir_config = dict(input_shape=[192, 256]) diff --git a/configs/mmseg/mmseg_regression_test.yaml b/configs/mmseg/mmseg_regression_test.yaml new file mode 100644 index 000000000..bf255b056 --- /dev/null +++ b/configs/mmseg/mmseg_regression_test.yaml @@ -0,0 +1,342 @@ +globals: + codebase_name: mmseg + codebase_dir: ../mmsegmentation + checkpoint_force_download: False + checkpoint_dir: ../mmdeploy_checkpoints + images: + img_leftImg8bit: &img_leftImg8bit ../mmsegmentation/tests/data/pseudo_cityscapes_dataset/leftImg8bit/frankfurt_000000_000294_leftImg8bit.png + img_loveda_0: &img_loveda_0 ../mmsegmentation/tests/data/pseudo_loveda_dataset/img_dir/0.png + img_loveda_1: &img_loveda_1 ../mmsegmentation/tests/data/pseudo_loveda_dataset/img_dir/1.png + img_loveda_2: &img_loveda_2 ../mmsegmentation/tests/data/pseudo_loveda_dataset/img_dir/2.png + img_potsdam: &img_potsdam ../mmsegmentation/tests/data/pseudo_potsdam_dataset/img_dir/2_10_0_0_512_512.png + img_vaihingen: &img_vaihingen ../mmsegmentation/tests/data/pseudo_vaihingen_dataset/img_dir/area1_0_0_512_512.png + metric_info: &metric_info + mIoU: # named after metafile.Results.Metrics + eval_name: mIoU # test.py --metrics args + metric_key: mIoU # eval OrderedDict key name + tolerance: 5 # metric ±n% + task_name: Semantic Segmentation # metafile.Results.Task + dataset: Cityscapes # metafile.Results.Dataset + convert_image: &convert_image + input_img: *img_leftImg8bit + test_img: *img_loveda_0 + backend_test: &default_backend_test True + sdk: + sdk_dynamic: &sdk_dynamic configs/mmseg/segmentation_sdk_dynamic.py + +onnxruntime: + pipeline_ort_static_fp32: &pipeline_ort_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_onnxruntime_static-1024x2048.py + + pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32 + convert_image: *convert_image + deploy_config: configs/mmseg/segmentation_onnxruntime_dynamic.py + +tensorrt: + pipeline_trt_static_fp32: &pipeline_trt_static_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_tensorrt_static-1024x2048.py + + pipeline_trt_static_fp16: &pipeline_trt_static_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_tensorrt-fp16_static-1024x2048.py + + pipeline_trt_static_int8: &pipeline_trt_static_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_tensorrt-int8_static-1024x2048.py + + + pipeline_trt_dynamic_fp32: &pipeline_trt_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_tensorrt_dynamic-512x1024-2048x2048.py + + pipeline_trt_dynamic_fp16: &pipeline_trt_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_tensorrt-fp16_dynamic-512x1024-2048x2048.py + + pipeline_trt_dynamic_int8: &pipeline_trt_dynamic_int8 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_dynamic + deploy_config: configs/mmseg/segmentation_tensorrt-int8_dynamic-512x1024-2048x2048.py + +openvino: + pipeline_openvino_dynamic_fp32: &pipeline_openvino_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmseg/segmentation_openvino_dynamic-1024x2048.py + + pipeline_openvino_static_fp32: &pipeline_openvino_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmseg/segmentation_openvino_static-1024x2048.py + +ncnn: + pipeline_ncnn_static_fp32: &pipeline_ncnn_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmseg/segmentation_ncnn_static-512x512.py + +pplnn: + pipeline_pplnn_dynamic_fp32: &pipeline_pplnn_dynamic_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmseg/segmentation_pplnn_dynamic-1024x2048.py + + pipeline_pplnn_static_fp32: &pipeline_pplnn_static_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmseg/segmentation_pplnn_static-1024x2048.py + +torchscript: + pipeline_ts_fp32: &pipeline_ts_fp32 + convert_image: *convert_image + backend_test: False + deploy_config: configs/mmseg/segmentation_torchscript.py + + +models: + - name: FCN + metafile: configs/fcn/fcn.yml + model_configs: + - configs/fcn/fcn_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 +# - *pipeline_trt_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 +# - *pipeline_trt_dynamic_int8 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: PSPNet + metafile: configs/pspnet/pspnet.yml + model_configs: + - configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_static_fp32 + - *pipeline_openvino_static_fp32 + + - name: deeplabv3 + metafile: configs/deeplabv3/deeplabv3.yml + model_configs: + - configs/deeplabv3/deeplabv3_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: deeplabv3+ + metafile: configs/deeplabv3plus/deeplabv3plus.yml + model_configs: + - configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: Fast-SCNN + metafile: configs/fastscnn/fastscnn.yml + model_configs: + - configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_pplnn_static_fp32 + - *pipeline_openvino_static_fp32 + + - name: UNet + metafile: configs/unet/unet.yml + model_configs: + - configs/unet/fcn_unet_s5-d16_4x4_512x1024_160k_cityscapes.py + pipelines: + - *pipeline_ts_fp32 + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_pplnn_dynamic_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: ANN + metafile: configs/ann/ann.yml + model_configs: + - configs/ann/ann_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + + - name: APCNet + metafile: configs/apcnet/apcnet.yml + model_configs: + - configs/apcnet/apcnet_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + + - name: BiSeNetV1 + metafile: configs/bisenetv1/bisenetv1.yml + model_configs: + - configs/bisenetv1/bisenetv1_r18-d32_4x4_1024x1024_160k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: BiSeNetV2 + metafile: configs/bisenetv2/bisenetv2.yml + model_configs: + - configs/bisenetv2/bisenetv2_fcn_4x4_1024x1024_160k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: CGNet + metafile: configs/cgnet/cgnet.yml + model_configs: + - configs/cgnet/cgnet_512x1024_60k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: EMANet + metafile: configs/emanet/emanet.yml + model_configs: + - configs/emanet/emanet_r50-d8_512x1024_80k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_openvino_dynamic_fp32 + + - name: EncNet + metafile: configs/encnet/encnet.yml + model_configs: + - configs/encnet/encnet_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_openvino_dynamic_fp32 + + - name: ERFNet + metafile: configs/erfnet/erfnet.yml + model_configs: + - configs/erfnet/erfnet_fcn_4x4_512x1024_160k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: FastFCN + metafile: configs/fastfcn/fastfcn.yml + model_configs: + - configs/fastfcn/fastfcn_r50-d32_jpu_aspp_512x1024_80k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: GCNet + metafile: configs/gcnet/gcnet.yml + model_configs: + - configs/gcnet/gcnet_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + + - name: ICNet + metafile: configs/icnet/icnet.yml + model_configs: + - configs/icnet/icnet_r18-d8_832x832_80k_cityscapes.py + pipelines: + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 + - *pipeline_openvino_static_fp32 + + - name: ISANet + metafile: configs/isanet/isanet.yml + model_configs: + - configs/isanet/isanet_r50-d8_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_openvino_dynamic_fp32 + + - name: OCRNet + metafile: configs/ocrnet/ocrnet.yml + model_configs: + - configs/ocrnet/ocrnet_hr18s_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: PointRend + metafile: configs/point_rend/point_rend.yml + model_configs: + - configs/point_rend/pointrend_r50_512x1024_80k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_openvino_dynamic_fp32 + + - name: Semantic FPN + metafile: configs/sem_fpn/sem_fpn.yml + model_configs: + - configs/sem_fpn/fpn_r50_512x1024_80k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: STDC + metafile: configs/stdc/stdc.yml + model_configs: + - configs/stdc/stdc1_in1k-pre_512x1024_80k_cityscapes.py + - configs/stdc/stdc2_in1k-pre_512x1024_80k_cityscapes.py + pipelines: + - *pipeline_ort_dynamic_fp32 + - *pipeline_trt_dynamic_fp16 + - *pipeline_ncnn_static_fp32 + - *pipeline_openvino_dynamic_fp32 + + - name: UPerNet + metafile: configs/upernet/upernet.yml + model_configs: + - configs/upernet/upernet_r50_512x1024_40k_cityscapes.py + pipelines: + - *pipeline_ort_static_fp32 + - *pipeline_trt_static_fp16 diff --git a/docs/zh_cn/tutorials/how_to_do_regression_test.md b/docs/zh_cn/tutorials/how_to_do_regression_test.md new file mode 100644 index 000000000..2a5011d80 --- /dev/null +++ b/docs/zh_cn/tutorials/how_to_do_regression_test.md @@ -0,0 +1,236 @@ +# 如何进行回归测试 + + +这篇教程介绍了如何进行回归测试。部署配置文件由`每个codebase的回归配置文件`，`推理框架配置信息`组成。 + + + +- [如何进行回归测试](#如何进行回归测试) + - [1. 用法](#1-用法) + - [参数解析](#参数解析) + - [示例](#示例) + - [2. 回归测试配置文件](#2-回归测试配置文件) + - [示例及参数解析](#示例及参数解析) + - [3. 生成的报告](#3-生成的报告) + - [模板](#模板) + - [示例](#示例) + - [4. 支持的后端](#4-支持的后端) + - [5. 支持的Codebase及其Metric](#5-支持的Codebase及其Metric) + - [6. 注意事项](#7-注意事项) + - [7. 常见问题](#8-常见问题) + + + +## 1. 用法 + +```shell +python ./tools/regression_test.py \ + --deploy-yml "${DEPLOY_YML_PATH}" \ + --backends "${BACKEND}" \ + --work-dir "${WORK_DIR}" \ + --device "${DEVICE}" \ + --log-level INFO \ + [--performance] +``` + +### 参数解析 + +- `--deploy-yml` : 需要测试的 codebase，eg.`configs/mmdet/mmdet_regression_test.yaml`，如果设置为 `all` 即全部测试。 +- `--backends` : 筛选测试的后端, 默认 `all`: 测全部`backend`, 也可传入若干个后端，例如 `onnxruntime tesnsorrt`。 +- `--work-dir` : 模型转换、报告生成的路径。 +- `--device` : 使用的设备，默认 `cuda`。 +- `--log-level` : 设置日记的等级，选项包括`'CRITICAL'， 'FATAL'， 'ERROR'， 'WARN'， 'WARNING'， 'INFO'， 'DEBUG'， 'NOTSET'`。默认是`INFO`。 +- `--performance` : 是否测试精度，加上则测试转换+精度，不加上则只测试转换 + +### 注意事项 +对于 Windows 用户： +1. 要在 shell 命令中使用 `&&` 连接符，需要下载并使用 `PowerShell 7 Preview 5+`。 +2. 如果您使用 conda env，可能需要在 regression_test.py 中将 `python3` 更改为 `python`，因为 `%USERPROFILE%\AppData\Local\Microsoft\WindowsApps` 目录中有 `python3.exe`。 + +## 例子 + +1. 测试 mmdet 和 mmpose 的所有 backend 的 转换+精度 + +```shell +python ./tools/regression_test.py \ + --deploy-yml ./configs/mmdet/mmdet_regression_test.yaml ./configs/mmpose/mmpose_regression_test.yaml \ + --backends all \ + --work-dir "../mmdeploy_regression_working_dir" \ + --device "cuda" \ + --log-level INFO \ + --performance +``` + +2. 测试 mmdet 和 mmpose 的某几个 backend 的 转换+精度 + +```shell +python ./tools/regression_test.py \ + --deploy-yml ./configs/mmdet/mmdet_regression_test.yaml ./configs/mmdet/mmpose.yaml \ + --backends onnxruntime tesnsorrt \ + --work-dir "../mmdeploy_regression_working_dir" \ + --device "cuda" \ + --log-level INFO \ + --performance +``` + +3. 测试 mmdet 和 mmpose 的某几个 backend，只需测试转换 + +```shell +python ./tools/regression_test.py \ + --deploy-yml ./configs/mmdet/mmdet_regression_test.yaml ./configs/mmdet/mmpose.yaml \ + --backends onnxruntime tesnsorrt \ + --work-dir "../mmdeploy_regression_working_dir" \ + --device "cuda" \ + --log-level INFO +``` + +## 2. 回归测试配置文件 + +### 示例及参数解析 + +```yaml +globals: + codebase_name: mmocr # 回归测试的 codebase 名称 + codebase_dir: ../mmocr # 回归测试的 codebase 路径 + checkpoint_force_download: False # 回归测试是否重新下载模型即使其已经存在 + checkpoint_dir: ../mmdeploy_checkpoints # 回归测试是否下载模型的路径 + images: # 测试使用图片 + img_224x224: &img_224x224 ./tests/data/tiger.jpeg + img_300x300: &img_300x300 + img_800x1344: &img_cityscapes_800x1344 + img_blank: &img_blank + metric_info: &metric_info # 指标参数 + hmean-iou: # 命名根据 metafile.Results.Metrics + eval_name: hmean-iou # 命名根据 test.py --metrics args 入参名称 + metric_key: 0_hmean-iou:hmean # 命名根据 eval 写入 log 的 key name + tolerance: 0.1 # 容忍的阈值区间 + task_name: Text Detection # 命名根据模型 metafile.Results.Task + dataset: ICDAR2015 #命名根据模型 metafile.Results.Dataset + word_acc: # 同上 + eval_name: acc + metric_key: 0_word_acc_ignore_case + tolerance: 0.2 + task_name: Text Recognition + dataset: IIIT5K + convert_image: &convert_image # 转换会使用到的图片 + input_img: *img_224x224 + test_img: *img_300x300 + backend_test: &default_backend_test True # 是否对 backend 进行精度测试 + sdk: # SDK 配置文件 + sdk_detection_dynamic: &sdk_detection_dynamic configs/mmocr/text-detection/text-detection_sdk_dynamic.py + sdk_recognition_dynamic: &sdk_recognition_dynamic configs/mmocr/text-recognition/text-recognition_sdk_dynamic.py + +onnxruntime: + pipeline_ort_recognition_static_fp32: &pipeline_ort_recognition_static_fp32 + convert_image: *convert_image # 转换过程中使用的图片 + backend_test: *default_backend_test # 是否进行后端测试，存在则判断，不存在则视为 False + sdk_config: *sdk_recognition_dynamic # 是否进行SDK测试，存在则使用特定的 SDK config 进行测试，不存在则视为不进行 SDK 测试 + deploy_config: configs/mmocr/text-recognition/text-recognition_onnxruntime_static.py # 使用的 deploy cfg 路径，基于 mmdeploy 的路径 + + pipeline_ort_recognition_dynamic_fp32: &pipeline_ort_recognition_dynamic_fp32 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_onnxruntime_dynamic.py + + pipeline_ort_detection_dynamic_fp32: &pipeline_ort_detection_dynamic_fp32 + convert_image: *convert_image + deploy_config: configs/mmocr/text-detection/text-detection_onnxruntime_dynamic.py + +tensorrt: + pipeline_trt_recognition_dynamic_fp16: &pipeline_trt_recognition_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_recognition_dynamic + deploy_config: configs/mmocr/text-recognition/text-recognition_tensorrt-fp16_dynamic-1x32x32-1x32x640.py + + pipeline_trt_detection_dynamic_fp16: &pipeline_trt_detection_dynamic_fp16 + convert_image: *convert_image + backend_test: *default_backend_test + sdk_config: *sdk_detection_dynamic + deploy_config: configs/mmocr/text-detection/text-detection_tensorrt-fp16_dynamic-320x320-1024x1824.py + +openvino: + # 此处省略，内容同上 +ncnn: + # 此处省略，内容同上 +pplnn: + # 此处省略，内容同上 +torchscript: + # 此处省略，内容同上 + + +models: + - name: crnn # 模型名称 + metafile: configs/textrecog/crnn/metafile.yml # 模型对应的 metafile 的路径，相对于 codebase 的路径 + codebase_model_config_dir: configs/textrecog/crnn # `model_configs` 的父文件夹路径，相对于 codebase 的路径 + model_configs: # 需要测试的 config 名称 + - crnn_academic_dataset.py + pipelines: # 使用的 pipeline + - *pipeline_ort_recognition_dynamic_fp32 + + - name: dbnet + metafile: configs/textdet/dbnet/metafile.yml + codebase_model_config_dir: configs/textdet/dbnet + model_configs: + - dbnet_r18_fpnc_1200e_icdar2015.py + pipelines: + - *pipeline_ort_detection_dynamic_fp32 + - *pipeline_trt_detection_dynamic_fp16 +``` + +## 3. 生成的报告 + +### 模板 + +|| model_name | model_config | task_name | model_checkpoint_name | dataset | backend_name | deploy_config | static_or_dynamic | precision_type | conversion_result | fps | metric_1 | metric_2 | metric_n | test_pass | +|------------|--------------|-----------------|-----------------------|----------|--------------|---------------|-------------------|----------------|-------------------|---|----------|----------|-----------|-----------|-----| +| 序号 | 模型名称 | model config 路径 | 执行的 task name | `.pth`模型路径 | 数据集名称 | 后端名称 | deploy cfg 路径 | 动态 or 静态 | 测试精度 | 模型转换结果 | FPS 数值 | 指标 1 数值 | 指标 2 数值 | 指标 n 数值 | 后端测试结果 | + +### 示例 + +这是 MMOCR 生成的报告 + +|| model_name | model_config | task_name | model_checkpoint_name | dataset | backend_name | deploy_config | static_or_dynamic | precision_type | conversion_result | fps | hmean-iou | word_acc | test_pass | +| ---- | ---------- | ------------------------------------------------------------ | ---------------- | ------------------------------------------------------------ | --------- | --------------- | ------------------------------------------------------------ | ----------------- | -------------- | ----------------- |-----------|----------|-----------| --------- | +| 0 | crnn | ../mmocr/configs/textrecog/crnn/crnn_academic_dataset.py | Text Recognition | ../mmdeploy_checkpoints/mmocr/crnn/crnn_academic-a723a1c5.pth | IIIT5K | Pytorch| -| - | - | - | - | - | 80.5 | -| +| 1 | crnn | ../mmocr/configs/textrecog/crnn/crnn_academic_dataset.py | Text Recognition | ${WORK_DIR}/mmocr/crnn/onnxruntime/static/crnn_academic-a723a1c5/end2end.onnx | x| onnxruntime | configs/mmocr/text-recognition/text-recognition_onnxruntime_dynamic.py | static | fp32 | True | 182.21 | - | 80.67 | True| +| 2 | crnn | ../mmocr/configs/textrecog/crnn/crnn_academic_dataset.py | Text Recognition | ${WORK_DIR}/mmocr/crnn/onnxruntime/static/crnn_academic-a723a1c5 | x| SDK-onnxruntime | configs/mmocr/text-recognition/text-recognition_sdk_dynamic.py | static | fp32 | True | x | - | x | False | +| 3 | dbnet| ../mmocr/configs/textdet/dbnet/dbnet_r18_fpnc_1200e_icdar2015.py | Text Detection | ../mmdeploy_checkpoints/mmocr/dbnet/dbnet_r18_fpnc_sbn_1200e_icdar2015_20210329-ba3ab597.pth | ICDAR2015 | Pytorch| -| - | - | - | - | 0.795 | - | -| +| 4 | dbnet| ../mmocr/configs/textdet/dbnet/dbnet_r18_fpnc_1200e_icdar2015.py | Text Detection | ../mmdeploy_checkpoints/mmocr/dbnet/dbnet_r18_fpnc_sbn_1200e_icdar2015_20210329-ba3ab597.pth | ICDAR | onnxruntime | configs/mmocr/text-detection/text-detection_onnxruntime_dynamic.py | dynamic | fp32 | True | - | - | - | True| +| 5 | dbnet| ../mmocr/configs/textdet/dbnet/dbnet_r18_fpnc_1200e_icdar2015.py | Text Detection | ${WORK_DIR}/mmocr/dbnet/tensorrt/dynamic/dbnet_r18_fpnc_sbn_1200e_icdar2015_20210329-ba3ab597/end2end.engine | ICDAR | tensorrt | configs/mmocr/text-detection/text-detection_tensorrt-fp16_dynamic-320x320-1024x1824.py | dynamic | fp16 | True | 229.06 | 0.793302 | - | True| +| 6 | dbnet| ../mmocr/configs/textdet/dbnet/dbnet_r18_fpnc_1200e_icdar2015.py | Text Detection | ${WORK_DIR}/mmocr/dbnet/tensorrt/dynamic/dbnet_r18_fpnc_sbn_1200e_icdar2015_20210329-ba3ab597 | ICDAR | SDK-tensorrt | configs/mmocr/text-detection/text-detection_sdk_dynamic.py | dynamic | fp16 | True | 140.06 | 0.795073 | - | True| + +## 4. 支持的后端 +- [x] ONNX Runtime +- [x] TensorRT +- [x] PPLNN +- [x] ncnn +- [x] OpenVINO +- [x] TorchScript +- [x] MMDeploy SDK + +## 5. 支持的Codebase及其Metric +- [x] mmdet + - [x] bbox +- [x] mmcls + - [x] accuracy +- [x] mmseg + - [x] mIoU +- [x] mmpose + - [x] AR + - [x] AP +- [x] mmocr + - [x] hmean + - [x] acc +- [x] mmedit + - [x] PSNR + - [x] SSIM + +## 6. 注意事项 + +暂无 + +## 7. 常见问题 + +暂无 diff --git a/requirements/tests.txt b/requirements/tests.txt index b3c09f586..afcf36128 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -3,5 +3,8 @@ coverage flake8 interrogate isort==4.3.21 +openpyxl +pandas pytest +xlrd==1.2.0 yapf diff --git a/tools/regression_test.py b/tools/regression_test.py new file mode 100644 index 000000000..15437a4b4 --- /dev/null +++ b/tools/regression_test.py @@ -0,0 +1,1162 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import logging +import os +from collections import OrderedDict +from pathlib import Path + +import mmcv +import openpyxl +import pandas as pd +import yaml +from torch.hub import download_url_to_file +from torch.multiprocessing import set_start_method + +import mmdeploy.version +from mmdeploy.utils import (get_backend, get_codebase, get_root_logger, + is_dynamic_shape, load_config) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Regression Test') + parser.add_argument( + '--deploy-yml', + nargs='+', + help='regression test yaml path.', + default=[ + './configs/mmcls/mmcls_regression_test.yaml', + './configs/mmdet/mmdet_regression_test.yaml', + './configs/mmseg/mmseg_regression_test.yaml', + './configs/mmpose/mmpose_regression_test.yaml', + './configs/mmocr/mmocr_regression_test.yaml', + './configs/mmedit/mmedit_regression_test.yaml' + ]) + parser.add_argument( + '--performance', + default=False, + action='store_true', + help='test performance if it set') + parser.add_argument( + '--backends', + nargs='+', + help='test specific backend(s)', + default=['all']) + parser.add_argument( + '--work-dir', + type=str, + help='the dir to save logs and models', + default='../mmdeploy_regression_working_dir') + parser.add_argument( + '--device', type=str, help='Device type, cuda or cpu', default='cuda') + parser.add_argument( + '--log-level', + help='set log level', + default='INFO', + choices=list(logging._nameToLevel.keys())) + args = parser.parse_args() + + return args + + +def merge_report(work_dir: str, logger: logging.Logger): + """Merge all the report into one report. + + Args: + work_dir (str): Work dir that including all reports. + logger (logging.Logger): Logger handler. + """ + work_dir = Path(work_dir) + res_file = work_dir.joinpath( + f'mmdeploy_regression_test_{mmdeploy.version.__version__}.xlsx') + logger.info(f'Whole result report saving in {res_file}') + + if res_file.exists(): + # delete if it existed + res_file.unlink() + + for report_file in work_dir.iterdir(): + if '_report.xlsx' not in report_file.name or \ + report_file.name == res_file.name or \ + not report_file.is_file(): + # skip other file + continue + # get info from report + logger.info(f'Merging {report_file}') + df = pd.read_excel(str(report_file)) + df.rename(columns={'Unnamed: 0': 'Index'}, inplace=True) + + # get key then convert to list + keys = list(df.keys()) + values = df.values.tolist() + + # sheet name + sheet_name = report_file.stem.split('_')[0] + + # begin to write + if res_file.exists(): + # load if it existed + wb = openpyxl.load_workbook(str(res_file)) + else: + wb = openpyxl.Workbook() + + # delete if sheet already exist + if sheet_name in wb.sheetnames: + wb.remove_sheet(wb[sheet_name]) + # create sheet + wb.create_sheet(title=sheet_name, index=0) + # write in row + wb[sheet_name].append(keys) + for value in values: + wb[sheet_name].append(value) + # delete the blank sheet + for name in wb.sheetnames: + ws = wb[name] + if ws.cell(1, 1).value is None: + wb.remove_sheet(ws) + # save to file + wb.save(str(res_file)) + + +def get_model_metafile_info(global_info: dict, model_info: dict, + logger: logging.Logger): + """Get model metafile information. + + Args: + global_info (dict): global info from deploy yaml. + model_info (dict): model info from deploy yaml. + logger (logging.Logger): Logger handler. + + Returns: + Dict: Meta info of each model config + """ + + # get info from global_info and model_info + checkpoint_dir = global_info.get('checkpoint_dir', None) + assert checkpoint_dir is not None + + codebase_dir = global_info.get('codebase_dir', None) + assert codebase_dir is not None + + codebase_name = global_info.get('codebase_name', None) + assert codebase_name is not None + + model_config_files = model_info.get('model_configs', []) + assert len(model_config_files) > 0 + + # make checkpoint save directory + checkpoint_save_dir = Path(checkpoint_dir).joinpath( + codebase_name, model_info.get('name')) + checkpoint_save_dir.mkdir(parents=True, exist_ok=True) + + # get model metafile info + metafile_path = Path(codebase_dir).joinpath(model_info.get('metafile')) + with open(metafile_path) as f: + metafile_info = yaml.load(f, Loader=yaml.FullLoader) + + model_meta_info = dict() + for meta_model in metafile_info.get('Models'): + if str(meta_model.get('Config')) not in model_config_files: + # skip if the model not in model_config_files + logger.warning(f'{meta_model.get("Config")} ' + f'not in {model_config_files}, pls check ! ' + 'Skip it...') + continue + + # get meta info + model_meta_info.update({meta_model.get('Config'): meta_model}) + + # get weight url + weights_url = meta_model.get('Weights') + weights_name = str(weights_url).split('/')[-1] + weights_save_path = checkpoint_save_dir.joinpath(weights_name) + if weights_save_path.exists() and \ + not global_info.get('checkpoint_force_download', False): + logger.info(f'model {weights_name} exist, skip download it...') + continue + + # Download weight + logger.info(f'Downloading {weights_url} to {weights_save_path}') + download_url_to_file( + weights_url, str(weights_save_path), progress=True) + + # check weather the weight download successful + if not weights_save_path.exists(): + raise FileExistsError(f'Weight {weights_name} download fail') + + logger.info('All models had been downloaded successful !') + return model_meta_info, checkpoint_save_dir, codebase_dir + + +def update_report(report_dict: dict, model_name: str, model_config: str, + task_name: str, model_checkpoint_name: str, dataset: str, + backend_name: str, deploy_config: str, + static_or_dynamic: str, precision_type: str, + conversion_result: str, fps: str, metric_info: list, + test_pass: str, report_txt_path: Path): + """Update report information. + + Args: + report_dict (dict): Report info dict. + model_name (str): Model name. + model_config (str): Model config name. + task_name (str): Task name. + model_checkpoint_name (str): Model checkpoint name. + dataset (str): Dataset name. + backend_name (str): Backend name. + deploy_config (str): Deploy config name. + static_or_dynamic (str): Static or dynamic. + precision_type (str): Precision type of the model. + conversion_result (str): Conversion result: Successful or Fail. + fps (str): Inference speed (ms/im). + metric_info (list): Metric info list of the ${modelName}.yml. + test_pass (str): Test result: Pass or Fail. + report_txt_path (Path): Report txt path. + """ + if '.pth' not in model_checkpoint_name: + # make model path shorter by cutting the work_dir_root + work_dir_root = report_txt_path.parent.absolute().resolve() + + if ' ' not in model_checkpoint_name: + model_checkpoint_name = \ + Path(model_checkpoint_name).absolute().resolve() + + model_checkpoint_name = \ + str(model_checkpoint_name).replace(str(work_dir_root), + '${WORK_DIR}') + + # save to tmp file + tmp_str = f'{model_name},{model_config},{task_name},' \ + f'{model_checkpoint_name},{dataset},{backend_name},' \ + f'{deploy_config},{static_or_dynamic},{precision_type},' \ + f'{conversion_result},{fps},' + + # save to report + report_dict.get('model_name').append(model_name) + report_dict.get('model_config').append(model_config) + report_dict.get('task_name').append(task_name) + report_dict.get('model_checkpoint_name').append(model_checkpoint_name) + report_dict.get('dataset').append(dataset) + report_dict.get('backend_name').append(backend_name) + report_dict.get('deploy_config').append(deploy_config) + report_dict.get('static_or_dynamic').append(static_or_dynamic) + report_dict.get('precision_type').append(precision_type) + report_dict.get('conversion_result').append(conversion_result) + report_dict.get('fps').append(fps) + + for metric in metric_info: + for metric_name, metric_value in metric.items(): + metric_name = str(metric_name) + report_dict.get(metric_name).append(metric_value) + tmp_str += f'{metric_value},' + report_dict.get('test_pass').append(test_pass) + + tmp_str += f'{test_pass}\n' + + with open(report_txt_path, 'a+', encoding='utf-8') as f: + f.write(tmp_str) + + +def get_pytorch_result(model_name: str, meta_info: dict, checkpoint_path: Path, + model_config_path: Path, model_config_name: str, + test_yaml_metric_info: dict, report_dict: dict, + logger: logging.Logger, report_txt_path: Path): + """Get metric from metafile info of the model. + + Args: + model_name (str): Name of model. + meta_info (dict): Metafile info from model's metafile.yml. + checkpoint_path (Path): Checkpoint path. + model_config_path (Path): Model config path. + model_config_name (str): Name of model config in meta_info. + test_yaml_metric_info (dict): Metrics info from test yaml. + report_dict (dict): Report info dict. + logger (logging.Logger): Logger. + report_txt_path (Path): Report txt path. + + Returns: + Dict: metric info of the model + """ + + if model_config_name not in meta_info: + logger.warning( + f'{model_config_name} not in meta_info, which is {meta_info}') + return {} + + # get metric + model_info = meta_info.get(model_config_name, None) + metafile_metric_info = model_info.get('Results', None) + + metric_list = [] + pytorch_metric = dict() + dataset_type = '' + task_type = '' + + # Get dataset + using_dataset = dict() + for _, v in test_yaml_metric_info.items(): + if v.get('dataset') is None: + continue + dataset_tmp = using_dataset.get(v.get('dataset'), []) + if v.get('task_name') not in dataset_tmp: + dataset_tmp.append(v.get('task_name')) + using_dataset.update({v.get('dataset'): dataset_tmp}) + + # Get metrics info from metafile + for metafile_metric in metafile_metric_info: + pytorch_meta_metric = metafile_metric.get('Metrics') + + dataset = metafile_metric.get('Dataset', '') + task_name = metafile_metric.get('Task', '') + + if task_name == 'Restorers': + # mmedit + dataset = 'Set5' + + if (len(using_dataset) > 1) and (dataset not in using_dataset): + logger.info(f'dataset not in {using_dataset}, skip it...') + continue + dataset_type += f'{dataset} | ' + + if task_name not in using_dataset.get(dataset): + # only add the metric with the correct dataset + logger.info(f'task_name ({task_name}) is not in' + f'{using_dataset.get(dataset)}, skip it...') + continue + task_type += f'{task_name} | ' + + # remove some metric which not in metric_info from test yaml + for k, v in pytorch_meta_metric.items(): + + if k not in test_yaml_metric_info and \ + 'Restorers' not in task_type: + continue + + if 'Restorers' in task_type and k not in dataset_type: + # mmedit + continue + + if isinstance(v, dict): + # mmedit + for sub_k, sub_v in v.items(): + use_metric = {sub_k: sub_v} + metric_list.append(use_metric) + pytorch_metric.update(use_metric) + else: + use_metric = {k: v} + metric_list.append(use_metric) + pytorch_metric.update(use_metric) + + dataset_type = dataset_type[:-3].upper() # remove the final ' | ' + task_type = task_type[:-3] # remove the final ' | ' + + # update useless metric + metric_all_list = [str(metric) for metric in test_yaml_metric_info] + metric_useless = set(metric_all_list) - set( + [str(metric) for metric in pytorch_metric]) + for metric in metric_useless: + metric_list.append({metric: '-'}) + + # get pytorch fps value + fps_info = model_info.get('Metadata').get('inference time (ms/im)') + if fps_info is None: + fps = '-' + elif isinstance(fps_info, list): + fps = fps_info[0].get('value') + else: + fps = fps_info.get('value') + + logger.info(f'Got metric_list = {metric_list} ') + logger.info(f'Got pytorch_metric = {pytorch_metric} ') + + # update report + update_report( + report_dict=report_dict, + model_name=model_name, + model_config=str(model_config_path), + task_name=task_type, + model_checkpoint_name=str(checkpoint_path), + dataset=dataset_type, + backend_name='Pytorch', + deploy_config='-', + static_or_dynamic='-', + precision_type='-', + conversion_result='-', + fps=fps, + metric_info=metric_list, + test_pass='-', + report_txt_path=report_txt_path) + + logger.info(f'Got {model_config_path} metric: {pytorch_metric}') + return pytorch_metric, dataset_type + + +def get_info_from_log_file(info_type: str, log_path: Path, + yaml_metric_key: str, logger: logging.Logger): + """Get fps and metric result from log file. + + Args: + info_type (str): Get which type of info: 'FPS' or 'metric'. + log_path (Path): Logger path. + yaml_metric_key (str): Name of metric from yaml metric_key. + logger (logger.Logger): Logger handler. + + Returns: + Float: Info value which get from logger file. + """ + if log_path.exists(): + with open(log_path, 'r') as f_log: + lines = f_log.readlines() + else: + logger.warning(f'{log_path} do not exist !!!') + lines = [] + + if info_type == 'FPS' and len(lines) > 1: + # Get FPS + line_count = 0 + fps_sum = 0.00 + fps_lines = lines[1:11] + + for line in fps_lines: + if 'FPS' not in line: + continue + line_count += 1 + fps_sum += float(line.split(' ')[-2]) + if fps_sum > 0.00: + info_value = f'{fps_sum / line_count:.2f}' + else: + info_value = 'x' + + elif info_type == 'metric' and len(lines) > 1: + # To calculate the final line index + if lines[-1] != '' and lines[-1] != '\n': + line_index = -1 + else: + line_index = -2 + + if yaml_metric_key in ['accuracy_top-1', 'mIoU', 'Eval-PSNR']: + # info in last second line + # mmcls, mmseg, mmedit + metric_line = lines[line_index - 1] + elif yaml_metric_key == 'AP': + # info in last tenth line + # mmpose + metric_line = lines[line_index - 9] + elif yaml_metric_key == 'AR': + # info in last fifth line + # mmpose + metric_line = lines[line_index - 4] + else: + # info in final line + # mmdet + metric_line = lines[line_index] + logger.info(f'Got metric_line = {metric_line}') + + metric_str = \ + metric_line.replace('\n', '').replace('\r', '').split(' - ')[-1] + logger.info(f'Got metric_str = {metric_str}') + logger.info(f'Got metric_info = {yaml_metric_key}') + + if 'OrderedDict' in metric_str: + # mmdet + evaluate_result = eval(metric_str) + if not isinstance(evaluate_result, OrderedDict): + logger.warning(f'Got error metric_dict = {metric_str}') + return 'x' + metric = evaluate_result.get(yaml_metric_key, 0.00) * 100 + elif 'accuracy_top' in metric_str: + # mmcls + metric = eval(metric_str.split(': ')[-1]) + if metric <= 1: + metric *= 100 + elif yaml_metric_key == 'mIoU' and '|' in metric_str: + # mmseg + metric = eval(metric_str.strip().split('|')[2]) + if metric <= 1: + metric *= 100 + elif yaml_metric_key in ['AP', 'AR']: + # mmpose + metric = eval(metric_str.split(': ')[-1]) + elif yaml_metric_key == '0_word_acc_ignore_case' or \ + yaml_metric_key == '0_hmean-iou:hmean': + # mmocr + evaluate_result = eval(metric_str) + if not isinstance(evaluate_result, dict): + logger.warning(f'Got error metric_dict = {metric_str}') + return 'x' + metric = evaluate_result.get(yaml_metric_key, 0.00) + if yaml_metric_key == '0_word_acc_ignore_case': + metric *= 100 + elif yaml_metric_key in ['Eval-PSNR', 'Eval-SSIM']: + # mmedit + metric = eval(metric_str.split(': ')[-1]) + else: + metric = 'x' + info_value = metric + else: + info_value = 'x' + + return info_value + + +def compare_metric(metric_value: float, metric_name: str, pytorch_metric: dict, + metric_info: dict): + """Compare metric value with the pytorch metric value and the tolerance. + + Args: + metric_value (float): Metric value. + metric_name (str): metric name. + pytorch_metric (dict): Pytorch metric which get from metafile. + metric_info (dict): Metric info from test yaml. + + Returns: + Bool: If the test pass or not. + """ + if metric_value == 'x': + return False + + metric_pytorch = pytorch_metric.get(str(metric_name)) + tolerance_value = metric_info.get(metric_name, {}).get('tolerance', 0.00) + if (metric_value - tolerance_value) <= metric_pytorch <= \ + (metric_value + tolerance_value): + test_pass = True + else: + test_pass = False + return test_pass + + +def get_fps_metric(shell_res: int, pytorch_metric: dict, metric_key: str, + yaml_metric_info_name: str, log_path: Path, + metrics_eval_list: dict, metric_info: dict, + logger: logging.Logger): + """Get fps and metric. + + Args: + shell_res (int): Backend convert result: 0 is success. + pytorch_metric (dict): Metric info of pytorch metafile. + metric_key (str):Metric info. + yaml_metric_info_name (str): Name of metric info in test yaml. + log_path (Path): Logger path. + metrics_eval_list (dict): Metric list from test yaml. + metric_info (dict): Metric info. + logger (logger.Logger): Logger handler. + + Returns: + Float: fps: FPS of the model. + List: metric_list: metric result list. + Bool: test_pass: If the test pass or not. + """ + metric_list = [] + + # check if converted successes or not. + if shell_res != 0: + fps = 'x' + metric_value = 'x' + else: + # Got fps from log file + fps = get_info_from_log_file('FPS', log_path, metric_key, logger) + logger.info(f'Got fps = {fps}') + + # Got metric from log file + metric_value = get_info_from_log_file('metric', log_path, metric_key, + logger) + logger.info(f'Got metric = {metric_value}') + + if yaml_metric_info_name is None: + logger.error(f'metrics_eval_list: {metrics_eval_list} ' + 'has not metric name') + assert yaml_metric_info_name is not None + + metric_list.append({yaml_metric_info_name: metric_value}) + test_pass = compare_metric(metric_value, yaml_metric_info_name, + pytorch_metric, metric_info) + + # same eval_name and multi metric output in one test + if yaml_metric_info_name == 'Top 1 Accuracy': + # mmcls + yaml_metric_info_name = 'Top 5 Accuracy' + second_get_metric = True + elif yaml_metric_info_name == 'AP': + # mmpose + yaml_metric_info_name = 'AR' + second_get_metric = True + elif yaml_metric_info_name == 'PSNR': + # mmedit + yaml_metric_info_name = 'SSIM' + second_get_metric = True + else: + second_get_metric = False + + if second_get_metric: + metric_key = metric_info.get(yaml_metric_info_name).get('metric_key') + if shell_res != 0: + metric_value = 'x' + else: + metric_value = get_info_from_log_file('metric', log_path, + metric_key, logger) + metric_list.append({yaml_metric_info_name: metric_value}) + if test_pass: + test_pass = compare_metric(metric_value, yaml_metric_info_name, + pytorch_metric, metric_info) + + return fps, metric_list, test_pass + + +def get_backend_fps_metric(deploy_cfg_path: str, model_cfg_path: Path, + convert_checkpoint_path: str, device_type: str, + eval_name: str, logger: logging.Logger, + metrics_eval_list: dict, pytorch_metric: dict, + metric_info: dict, backend_name: str, + precision_type: str, metric_useless: set, + convert_result: bool, report_dict: dict, + infer_type: str, log_path: Path, dataset_type: str, + report_txt_path: Path, model_name: str): + """Get backend fps and metric. + + Args: + deploy_cfg_path (str): Deploy config path. + model_cfg_path (Path): Model config path. + convert_checkpoint_path (str): Converted checkpoint path. + device_type (str): Device for converting. + eval_name (str): Name of evaluation. + logger (logging.Logger): Logger handler. + metrics_eval_list (dict): Evaluation metric info dict. + pytorch_metric (dict): Pytorch metric info dict get from metafile. + metric_info (dict): Metric info from test yaml. + backend_name (str): Backend name. + precision_type (str): Precision type for evaluation. + metric_useless (set): Useless metric for specific the model. + convert_result (bool): Backend convert result. + report_dict (dict): Backend convert result. + infer_type (str): Infer type. + log_path (Path): Logger save path. + dataset_type (str): Dataset type. + report_txt_path (Path): report txt save path. + model_name (str): Name of model in test yaml. + """ + cmd_str = f'cd {str(Path(__file__).absolute().parent.parent)} && ' \ + 'python3 tools/test.py ' \ + f'{deploy_cfg_path} ' \ + f'{str(model_cfg_path.absolute())} ' \ + f'--model "{convert_checkpoint_path}" ' \ + f'--device {device_type} ' \ + f'--log2file "{log_path}" ' \ + f'--speed-test ' + + codebase_name = get_codebase(str(deploy_cfg_path)).value + if codebase_name != 'mmedit': + # mmedit dont --metric + cmd_str += f'--metrics {eval_name} ' + + logger.info(f'Process cmd = {cmd_str}') + + # Test backend + shell_res = os.system(cmd_str) + logger.info(f'Got shell_res = {shell_res}') + + metric_key = '' + metric_name = '' + task_name = '' + for key, value in metric_info.items(): + if value.get('eval_name', '') == eval_name: + metric_name = key + metric_key = value.get('metric_key', '') + task_name = value.get('task_name', '') + break + + logger.info(f'Got metric_name = {metric_name}') + logger.info(f'Got metric_key = {metric_key}') + + fps, metric_list, test_pass = \ + get_fps_metric(shell_res, pytorch_metric, metric_key, metric_name, + log_path, metrics_eval_list, metric_info, logger) + + # update useless metric + for metric in metric_useless: + metric_list.append({metric: '-'}) + + if len(metrics_eval_list) > 1 and codebase_name == 'mmdet': + # one model has more than one task, like Mask R-CNN + for name in pytorch_metric: + if name in metric_useless or name == metric_name: + continue + metric_list.append({name: '-'}) + + # update the report + update_report( + report_dict=report_dict, + model_name=model_name, + model_config=str(model_cfg_path), + task_name=task_name, + model_checkpoint_name=convert_checkpoint_path, + dataset=dataset_type, + backend_name=backend_name, + deploy_config=str(deploy_cfg_path), + static_or_dynamic=infer_type, + precision_type=precision_type, + conversion_result=str(convert_result), + fps=fps, + metric_info=metric_list, + test_pass=str(test_pass), + report_txt_path=report_txt_path) + + +def get_precision_type(deploy_cfg_name: str): + """Get backend precision_type according to the name of deploy config. + + Args: + deploy_cfg_name (str): Name of the deploy config. + + Returns: + Str: precision_type: Precision type of the deployment name. + """ + if 'int8' in deploy_cfg_name: + precision_type = 'int8' + elif 'fp16' in deploy_cfg_name: + precision_type = 'fp16' + else: + precision_type = 'fp32' + + return precision_type + + +def replace_top_in_pipeline_json(backend_output_path: Path, + logger: logging.Logger): + """Replace `topk` with `num_classes` in `pipeline.json`. + + Args: + backend_output_path (Path): Backend convert result path. + logger (logger.Logger): Logger handler. + """ + + sdk_pipeline_json_path = backend_output_path.joinpath('pipeline.json') + sdk_pipeline_json = mmcv.load(sdk_pipeline_json_path) + + pipeline_tasks = sdk_pipeline_json.get('pipeline', {}).get('tasks', []) + for index, task in enumerate(pipeline_tasks): + if task.get('name', '') != 'postprocess': + continue + num_classes = task.get('params', {}).get('num_classes', 0) + if 'topk' not in task.get('params', {}): + continue + sdk_pipeline_json['pipeline']['tasks'][index]['params']['topk'] = \ + num_classes + + logger.info(f'sdk_pipeline_json = {sdk_pipeline_json}') + + mmcv.dump( + sdk_pipeline_json, sdk_pipeline_json_path, sort_keys=False, indent=4) + + logger.info('replace done') + + +def gen_log_path(backend_output_path: Path, log_name: str): + log_path = backend_output_path.joinpath(log_name).absolute().resolve() + if log_path.exists(): + # clear the log file + with open(log_path, 'w') as f_log: + f_log.write('') + + return log_path + + +def get_backend_result(pipeline_info: dict, model_cfg_path: Path, + checkpoint_path: Path, work_dir: Path, device_type: str, + pytorch_metric: dict, metric_info: dict, + report_dict: dict, test_type: str, + logger: logging.Logger, backend_file_name: [str, list], + report_txt_path: Path, metafile_dataset: str, + model_name: str): + """Convert model to onnx and then get metric. + + Args: + pipeline_info (dict): Pipeline info of test yaml. + model_cfg_path (Path): Model config file path. + checkpoint_path (Path): Checkpoints path. + work_dir (Path): A working directory. + device_type (str): A string specifying device, defaults to 'cuda'. + pytorch_metric (dict): All pytorch metric info. + metric_info (dict): Metrics info. + report_dict (dict): Report info dict. + test_type (str): Test type. 'precision' or 'convert'. + logger (logging.Logger): Logger. + backend_file_name (str | list): backend file save name. + report_txt_path (Path): report txt path. + metafile_dataset (str): Dataset type get from metafile. + model_name (str): Name of model in test yaml. + """ + # get backend_test info + backend_test = pipeline_info.get('backend_test', False) + + # get convert_image info + convert_image_info = pipeline_info.get('convert_image', None) + if convert_image_info is not None: + input_img_path = \ + convert_image_info.get('input_img', './tests/data/tiger.jpeg') + test_img_path = convert_image_info.get('test_img', None) + else: + input_img_path = './tests/data/tiger.jpeg' + test_img_path = None + + # get sdk_cfg info + sdk_config = pipeline_info.get('sdk_config', None) + if sdk_config is not None: + sdk_config = Path(sdk_config) + + # Overwrite metric tolerance + metric_tolerance = pipeline_info.get('metric_tolerance', None) + if metric_tolerance is not None: + for metric, new_tolerance in metric_tolerance.items(): + if metric not in metric_info: + logger.debug(f'{metric} not in {metric_info},' + 'skip compare it...') + continue + if new_tolerance is None: + logger.debug('new_tolerance is None, skip it ...') + continue + metric_info[metric]['tolerance'] = new_tolerance + + if backend_test is False and sdk_config is None: + test_type = 'convert' + + metric_name_list = [str(metric) for metric in pytorch_metric] + assert len(metric_name_list) > 0 + metric_all_list = [str(metric) for metric in metric_info] + metric_useless = set(metric_all_list) - set(metric_name_list) + + deploy_cfg_path = Path(pipeline_info.get('deploy_config')) + backend_name = str(get_backend(str(deploy_cfg_path)).name).lower() + infer_type = \ + 'dynamic' if is_dynamic_shape(str(deploy_cfg_path)) else 'static' + + precision_type = get_precision_type(deploy_cfg_path.name) + codebase_name = get_codebase(str(deploy_cfg_path)).value + + backend_output_path = Path(work_dir). \ + joinpath(Path(checkpoint_path).parent.parent.name, + Path(checkpoint_path).parent.name, + backend_name, + infer_type, + Path(checkpoint_path).stem) + backend_output_path.mkdir(parents=True, exist_ok=True) + + # convert cmd string + cmd_str = f'cd {str(str(Path(__file__).absolute().parent.parent))} && ' \ + 'python3 ./tools/deploy.py ' \ + f'{str(deploy_cfg_path.absolute().resolve())} ' \ + f'{str(model_cfg_path.absolute().resolve())} ' \ + f'"{str(checkpoint_path.absolute().resolve())}" ' \ + f'"{input_img_path}" ' \ + f'--work-dir "{backend_output_path}" ' \ + f'--device {device_type} ' \ + '--log-level INFO' + + if sdk_config is not None: + cmd_str += ' --dump-info' + + if test_img_path is not None: + cmd_str += f' --test-img {test_img_path}' + + if precision_type == 'int8': + calib_dataset_cfg = pipeline_info.get('calib_dataset_cfg', None) + if calib_dataset_cfg is not None: + cmd_str += f' --calib-dataset-cfg {calib_dataset_cfg}' + + logger.info(f'Process cmd = {cmd_str}') + + # Convert the model to specific backend + shell_res = os.system(cmd_str) + logger.info(f'Got shell_res = {shell_res}') + + # check if converted successes or not. + if shell_res == 0: + convert_result = True + else: + convert_result = False + logger.info(f'Got convert_result = {convert_result}') + + if isinstance(backend_file_name, list): + convert_checkpoint_path = '' + for backend_file in backend_file_name: + backend_path = backend_output_path.joinpath(backend_file) + backend_path = backend_path.absolute().resolve() + convert_checkpoint_path += f'{str(backend_path)} ' + else: + convert_checkpoint_path = \ + str(backend_output_path.joinpath(backend_file_name)) + + # load deploy_cfg + deploy_cfg, model_cfg = \ + load_config(str(deploy_cfg_path), + str(model_cfg_path.absolute())) + # get dataset type + if 'dataset_type' in model_cfg: + dataset_type = \ + str(model_cfg.dataset_type).upper().replace('DATASET', '') + else: + dataset_type = metafile_dataset + + # Test the model + if convert_result and test_type == 'precision': + # Get evaluation metric from model config + metrics_eval_list = model_cfg.evaluation.get('metric', []) + if isinstance(metrics_eval_list, str): + # some config is using str only + metrics_eval_list = [metrics_eval_list] + + # assert len(metrics_eval_list) > 0 + logger.info(f'Got metrics_eval_list = {metrics_eval_list}') + if len(metrics_eval_list) == 0 and codebase_name == 'mmedit': + metrics_eval_list = ['PSNR'] + + # test the model metric + for metric_name in metrics_eval_list: + if backend_test: + log_path = \ + gen_log_path(backend_output_path, 'backend_test.log') + get_backend_fps_metric( + deploy_cfg_path=str(deploy_cfg_path), + model_cfg_path=model_cfg_path, + convert_checkpoint_path=convert_checkpoint_path, + device_type=device_type, + eval_name=metric_name, + logger=logger, + metrics_eval_list=metrics_eval_list, + pytorch_metric=pytorch_metric, + metric_info=metric_info, + backend_name=backend_name, + precision_type=precision_type, + metric_useless=metric_useless, + convert_result=convert_result, + report_dict=report_dict, + infer_type=infer_type, + log_path=log_path, + dataset_type=dataset_type, + report_txt_path=report_txt_path, + model_name=model_name) + + if sdk_config is not None: + + if codebase_name == 'mmcls': + replace_top_in_pipeline_json(backend_output_path, logger) + + log_path = gen_log_path(backend_output_path, 'sdk_test.log') + # sdk test + get_backend_fps_metric( + deploy_cfg_path=str(sdk_config), + model_cfg_path=model_cfg_path, + convert_checkpoint_path=str(backend_output_path), + device_type=device_type, + eval_name=metric_name, + logger=logger, + metrics_eval_list=metrics_eval_list, + pytorch_metric=pytorch_metric, + metric_info=metric_info, + backend_name=f'SDK-{backend_name}', + precision_type=precision_type, + metric_useless=metric_useless, + convert_result=convert_result, + report_dict=report_dict, + infer_type=infer_type, + log_path=log_path, + dataset_type=dataset_type, + report_txt_path=report_txt_path, + model_name=model_name) + else: + logger.info('Only test convert, saving to report...') + metric_list = [] + fps = '-' + + task_name = '' + for metric in metric_name_list: + metric_list.append({metric: '-'}) + metric_task_name = metric_info.get(metric, {}).get('task_name', '') + if metric_task_name in task_name: + logger.debug('metric_task_name exist, skip for adding it...') + continue + task_name += f'{metric_task_name} | ' + if ' | ' == task_name[-3:]: + task_name = task_name[:-3] + test_pass = True if convert_result else False + + # update useless metric + for metric in metric_useless: + metric_list.append({metric: '-'}) + + if convert_result: + report_checkpoint = convert_checkpoint_path + else: + report_checkpoint = str(checkpoint_path) + + # update the report + update_report( + report_dict=report_dict, + model_name=model_name, + model_config=str(model_cfg_path), + task_name=task_name, + model_checkpoint_name=report_checkpoint, + dataset=dataset_type, + backend_name=backend_name, + deploy_config=str(deploy_cfg_path), + static_or_dynamic=infer_type, + precision_type=precision_type, + conversion_result=str(convert_result), + fps=fps, + metric_info=metric_list, + test_pass=str(test_pass), + report_txt_path=report_txt_path) + + +def save_report(report_info: dict, report_save_path: Path, + logger: logging.Logger): + """Convert model to onnx and then get metric. + + Args: + report_info (dict): Report info dict. + report_save_path (Path): Report save path. + logger (logging.Logger): Logger. + """ + logger.info('Saving regression test report to ' + f'{report_save_path.absolute().resolve()}, pls wait...') + try: + df = pd.DataFrame(report_info) + df.to_excel(report_save_path) + except ValueError: + logger.info(f'Got error report_info = {report_info}') + + logger.info('Saved regression test report to ' + f'{report_save_path.absolute().resolve()}.') + + +def main(): + args = parse_args() + set_start_method('spawn') + logger = get_root_logger(log_level=args.log_level) + + test_type = 'precision' if args.performance else 'convert' + logger.info(f'Processing regression test in {test_type} mode.') + + backend_file_info = { + 'onnxruntime': 'end2end.onnx', + 'tensorrt': 'end2end.engine', + 'openvino': 'end2end.xml', + 'ncnn': ['end2end.param', 'end2end.bin'], + 'pplnn': ['end2end.onnx', 'end2end.json'], + 'torchscript': 'end2end.pt' + } + + backend_list = args.backends + if backend_list == ['all']: + backend_list = [ + 'onnxruntime', 'tensorrt', 'openvino', 'ncnn', 'pplnn', + 'torchscript' + ] + assert isinstance(backend_list, list) + logger.info(f'Regression test backend list = {backend_list}') + + work_dir = Path(args.work_dir) + work_dir.mkdir(parents=True, exist_ok=True) + + for deploy_yaml in args.deploy_yml: + + if not Path(deploy_yaml).exists(): + raise FileNotFoundError(f'deploy_yaml {deploy_yaml} not found, ' + 'please check !') + + with open(deploy_yaml) as f: + yaml_info = yaml.load(f, Loader=yaml.FullLoader) + + report_save_path = \ + work_dir.joinpath(Path(deploy_yaml).stem + '_report.xlsx') + report_txt_path = report_save_path.with_suffix('.txt') + + report_dict = { + 'model_name': [], + 'model_config': [], + 'task_name': [], + 'model_checkpoint_name': [], + 'dataset': [], + 'backend_name': [], + 'deploy_config': [], + 'static_or_dynamic': [], + 'precision_type': [], + 'conversion_result': [], + 'fps': [] + } + + global_info = yaml_info.get('globals') + for metric_name in global_info.get('metric_info', {}): + report_dict.update({metric_name: []}) + metric_info = global_info.get('metric_info', {}) + report_dict.update({'test_pass': []}) + + with open(report_txt_path, 'w') as f_report: + title_str = '' + for key in report_dict: + title_str += f'{key},' + title_str = title_str[:-1] + '\n' + f_report.write(title_str) # clear the report tmp file + + models_info = yaml_info.get('models') + for models in models_info: + if 'model_configs' not in models: + logger.warning('Can not find field "model_configs", ' + f'skipping {models.get("name")}...') + continue + + model_metafile_info, checkpoint_save_dir, codebase_dir = \ + get_model_metafile_info(global_info, models, logger) + for model_config in model_metafile_info: + logger.info(f'Processing test for {model_config}...') + + # Get backends info + pipelines_info = models.get('pipelines', None) + if pipelines_info is None: + logger.warning('pipelines_info is None, skip it...') + continue + + # Get model config path + model_cfg_path = Path(codebase_dir).joinpath(model_config) + assert model_cfg_path.exists() + + # Get checkpoint path + checkpoint_name = Path( + model_metafile_info.get(model_config).get('Weights')).name + checkpoint_path = Path(checkpoint_save_dir, checkpoint_name) + assert checkpoint_path.exists() + + # Get pytorch from metafile.yml + pytorch_metric, metafile_dataset = get_pytorch_result( + models.get('name'), model_metafile_info, checkpoint_path, + model_cfg_path, model_config, metric_info, report_dict, + logger, report_txt_path) + + for pipeline in pipelines_info: + deploy_config = pipeline.get('deploy_config') + backend_name = get_backend(deploy_config).name.lower() + if backend_name not in backend_list: + logger.warning(f'backend_name ({backend_name}) not ' + f'in {backend_list}, skip it...') + continue + + backend_file_name = \ + backend_file_info.get(backend_name, None) + if backend_file_name is None: + logger.warning('backend_file_name is None, ' + 'skip it...') + continue + + get_backend_result(pipeline, model_cfg_path, + checkpoint_path, work_dir, args.device, + pytorch_metric, metric_info, + report_dict, test_type, logger, + backend_file_name, report_txt_path, + metafile_dataset, models.get('name')) + + save_report(report_dict, report_save_path, logger) + + # merge report + merge_report(str(work_dir), logger) + + +if __name__ == '__main__': + main()