v3.1

add layer-wise operation profiling util-tools

MNN

bin/timeProfile.out .mnn/fp16/LeViT_128.mnn 100 0 1x3x224x224 1 2
bin/timeProfile.out .mnn/int8/LeViT_128.mnn 100 0 1x3x224x224 1 2
cd .mnn; ln -sf fp16/*.mnn .; cd ..
DEF="-DUSE_PERF" FP=16 BACK=x MODEL=128 make run-mnn-perf

cd .mnn; ln -sf int8/*.mnn .; cd ..
DEF="-DUSE_PERF" BACK=z MODEL=128 make run-mnn-perf

TFLite

bazel build -c opt //tensorflow/lite/tools/benchmark:benchmark_model

bin/benchmark_model \
  --graph=.tflite/tinynn-32/LeViT_128.tflite \
  --num_threads=1 \
  --warmup_runs=10 \
  --num_runs=50 \
  --enable_op_profiling=true \
  --use_xnnpack=true \
  --xnnpack_force_fp16=true

bin/benchmark_model \
  --graph=.tflite/tinynn-d8/LeViT_128.tflite \
  --num_threads=1 \
  --warmup_runs=10 \
  --num_runs=50 \
  --enable_op_profiling=true

cd .tflite; ln -sf tinynn-32/*.tflite .; cd ..
DEF="-DUSE_PERF" FP=16 BACK=x MODEL=128 make run-tflite-perf

cd .tflite; ln -sf tinynn-d8/*.tflite .; cd ..
DEF="-DUSE_PERF" BACK=z MODEL=128 make run-tflite-perf

TensorRT

/usr/src/tensorrt/bin/trtexec --batch=1 --loadEngine=.onnx/gpu-fp16/mobilevit_x_small.engine --dumpProfile --separateProfileRun | tee /tmp/fp16
python python/utils/trt-op-statistic.py fp16

/usr/src/tensorrt/bin/trtexec --batch=1 --loadEngine=.onnx/gpu-int8/mobilevit_x_small.engine --dumpProfile --separateProfileRun | tee /tmp/int8
python python/utils/trt-op-statistic.py int8

CANN

python python/cann-perf.py --format fp16 --only-test mobilevit_x_small --profile 2>/dev/null | tee /tmp/fp16
python python/utils/acl-op-statistic.py fp16

python python/cann-perf.py --format int8 --only-test mobilevit_x_small --profile 2>/dev/null | tee /tmp/int8
python python/utils/acl-op-statistic.py int8

OpenVino

export LD_LIBRARY_PATH=xxx
bin/benchmark_app -m .xml/fp16/mobilevit_x_small.xml -t 20 -pc -d NPU -report_type detailed_counters -hint latency -api sync -infer_precision f16
mv benchmark_detailed_counters_report.csv /tmp/fp16.csv; python /tmp/opv-op-statistic.py fp16

bin/benchmark_app -m .xml/int8/mobilevit_x_small.xml -t 20 -pc -d NPU -report_type detailed_counters -hint latency -api sync
mv benchmark_detailed_counters_report.csv /tmp/int8.csv; python /tmp/opv-op-statistic.py int8 

the openvino benchmark_app is modified by myself with addition of my NPU config(which has better performance)

diff --git a/samples/cpp/benchmark_app/main.cpp b/samples/cpp/benchmark_app/main.cpp
index 2cfd15b77a..00dbf04752 100644
--- a/samples/cpp/benchmark_app/main.cpp
+++ b/samples/cpp/benchmark_app/main.cpp
@@ -13,6 +13,7 @@
 // clang-format off
 #include "openvino/openvino.hpp"
 #include "openvino/pass/serialize.hpp"
+#include "openvino/runtime/intel_npu/properties.hpp"
 
 #ifndef IN_OV_COMPONENT
 #    define IN_OV_COMPONENT
@@ -742,6 +743,14 @@ int main(int argc, char* argv[]) {
             // ----------------- 7. Loading the model to the device
             // --------------------------------------------------------
             next_step();
+            if (device_name == "NPU") {
+                device_config = {};
+                device_config.insert(ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY));
+                device_config.insert(ov::intel_npu::compilation_mode_params(ov::Any("optimization-level=2 performance-hint-override=latency")));
+                device_config.insert(ov::intel_npu::turbo(true));
+                device_config.insert(ov::hint::inference_precision(ov::element::f16));
+                device_config[ov::enable_profiling.name()] = FLAGS_pc;
+            }
             startTime = Time::now();
             compiledModel = core.compile_model(model, device_name, device_config);
             duration_ms = get_duration_ms_till_now(startTime);