INC New API TF oob, wide_deep_large_ds, 3dunet-mlperf examples

.azure-pipelines/model-test.yml

@@ -10,6 +10,7 @@ pr:
     include:
       - neural_compressor
       - setup.py
+      - examples/tensorflow/oob_models/quantization/ptq
     exclude:
       - neural_compressor/ux
 

.azure-pipelines/scripts/models/env_setup.sh

@@ -37,6 +37,9 @@ for i in "$@"; do
     --new_benchmark=*)
         new_benchmark=$(echo $i | sed "s/${PATTERN}//")
         ;;
+    --inc_new_api=*)
+        inc_new_api=$(echo $i | sed "s/${PATTERN}//")
+        ;;
     *)
         echo "Parameter $i not recognized."
         exit 1
@@ -61,7 +64,19 @@ fi
 $BOLD_YELLOW && echo "====== install requirements ======" && $RESET
 /bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh
 
-cd ${WORK_SOURCE_DIR}/${model_src_dir}
+cd ${WORK_SOURCE_DIR}
+if [[ "${inc_new_api}" == "false" ]]; then
+    echo "copy old api examples to workspace..."
+    git clone -b old_api_examples https://github.com/intel/neural-compressor.git old-lpot-models
+    cd old-lpot-models
+    git branch
+    cd -
+    rm -rf ${model_src_dir}
+    mkdir -p ${model_src_dir}
+    cp -r old-lpot-models/examples/${framework}/${model_src_dir} ${WORK_SOURCE_DIR}/${model_src_dir}/../
+fi
+
+cd ${model_src_dir}
 pip install ruamel_yaml
 pip install psutil
 pip install protobuf==3.20.1
@@ -113,16 +128,18 @@ else
     $BOLD_RED && echo "Not found requirements.txt file." && $RESET
 fi
 
-$BOLD_YELLOW && echo "======== update yaml config ========" && $RESET
-$BOLD_YELLOW && echo -e "\nPrint origin yaml..." && $RESET
-cat ${yaml}
-python ${SCRIPTS_PATH}/update_yaml_config.py \
-    --yaml=${yaml} \
-    --framework=${framework} \
-    --dataset_location=${dataset_location} \
-    --batch_size=${batch_size} \
-    --strategy=${strategy} \
-    --new_benchmark=${new_benchmark} \
-    --multi_instance='true'
-$BOLD_YELLOW && echo -e "\nPrint updated yaml... " && $RESET
-cat ${yaml}
+if [[ "${inc_new_api}" == "false" ]]; then
+    $BOLD_YELLOW && echo "======== update yaml config ========" && $RESET
+    $BOLD_YELLOW && echo -e "\nPrint origin yaml..." && $RESET
+    cat ${yaml}
+    python ${SCRIPTS_PATH}/update_yaml_config.py \
+        --yaml=${yaml} \
+        --framework=${framework} \
+        --dataset_location=${dataset_location} \
+        --batch_size=${batch_size} \
+        --strategy=${strategy} \
+        --new_benchmark=${new_benchmark} \
+        --multi_instance='true'
+    $BOLD_YELLOW && echo -e "\nPrint updated yaml... " && $RESET
+    cat ${yaml}
+fi

.azure-pipelines/scripts/models/run_model_trigger_common.sh

@@ -29,6 +29,8 @@ do
             strategy=`echo $i | sed "s/${PATTERN}//"`;;
         --new_benchmark=*)
             new_benchmark=`echo $i | sed "s/${PATTERN}//"`;;
+        --inc_new_api=*)
+            inc_new_api=`echo $i | sed "s/${PATTERN}//"`;;
         --tuning_cmd=*)
             tuning_cmd=`echo $i | sed "s/${PATTERN}//"`;;
         --benchmark_cmd=*)
@@ -62,7 +64,8 @@ if [ "${mode}" == "env_setup" ]; then
         --dataset_location=${dataset_location} \
         --batch_size=${batch_size} \
         --strategy=${strategy} \
-        --new_benchmark=${new_benchmark}
+        --new_benchmark=${new_benchmark} \
+        --inc_new_api="${inc_new_api}"
 elif [ "${mode}" == "tuning" ]; then
     cd ${WORK_SOURCE_DIR}/${model_src_dir}
     $BOLD_YELLOW && echo "workspace ${WORK_SOURCE_DIR}/${model_src_dir}" && $RESET

.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh

@@ -24,7 +24,7 @@ done
 FRAMEWORK="mxnet"
 FRAMEWORK_VERSION="1.9.1"
 
-
+inc_new_api=false
 # ======== set up config for mxnet models ========
 if [ "${model}" == "resnet50v1" ]; then
     model_src_dir="image_recognition/cnn_models/quantization/ptq"

.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh

@@ -24,7 +24,7 @@ done
 FRAMEWORK="onnxrt"
 FRAMEWORK_VERSION="1.13.1"
 
-
+inc_new_api=false
 # ======== set up config for onnxrt models ========
 if [ "${model}" == "resnet50-v1-12" ]; then
     model_src_dir="image_recognition/onnx_model_zoo/resnet50/quantization/ptq"
@@ -82,6 +82,7 @@ fi
     --new_benchmark=${new_benchmark} \
     --tuning_cmd="${tuning_cmd}" \
     --benchmark_cmd="${benchmark_cmd}" \
+    --inc_new_api="${inc_new_api}" \
     --mode=${mode} \
     --USE_TUNE_ACC=${USE_TUNE_ACC} \
     --PERF_STABLE_CHECK=${PERF_STABLE_CHECK} \

.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh

@@ -25,7 +25,7 @@ FRAMEWORK="pytorch"
 FRAMEWORK_VERSION="1.12.0+cpu"
 TORCH_VISION_VERSION="0.13.0+cpu"
 
-
+inc_new_api=false
 # ======== set up config for pytorch models ========
 if [ "${model}" == "resnet18" ]; then
     model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/eager"
@@ -64,6 +64,7 @@ fi
     --new_benchmark=${new_benchmark} \
     --tuning_cmd="${tuning_cmd}" \
     --benchmark_cmd="${benchmark_cmd}" \
+    --inc_new_api="${inc_new_api}" \
     --mode=${mode} \
     --USE_TUNE_ACC=${USE_TUNE_ACC} \
     --PERF_STABLE_CHECK=${PERF_STABLE_CHECK} \

.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh

@@ -24,6 +24,7 @@ done
 FRAMEWORK="tensorflow"
 FRAMEWORK_VERSION="2.10.0"
 
+inc_new_api=false
 # ======== set up config for tensorflow models ========
 if [ "${model}" == "resnet50v1.5" ]; then
     model_src_dir="image_recognition/tensorflow_models/quantization/ptq"
@@ -83,8 +84,9 @@ elif [ "${model}" == "darknet19" ]; then
     strategy="basic"
     batch_size=1
     new_benchmark=false
-    tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}"
-    benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500"
+    inc_new_api=true
+    tuning_cmd="bash run_tuning.sh --topology=${model} --input_model=${input_model}"
+    benchmark_cmd="bash run_benchmark.sh --topology=${model} --mode=performance --batch_size=1 --iters=500"
 elif [ "${model}" == "densenet-121" ]; then
     model_src_dir="oob_models/quantization/ptq"
     dataset_location=""
@@ -93,8 +95,9 @@ elif [ "${model}" == "densenet-121" ]; then
     strategy="basic"
     batch_size=1
     new_benchmark=false
-    tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}"
-    benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500"
+    inc_new_api=true
+    tuning_cmd="bash run_tuning.sh --topology=${model} --input_model=${input_model}"
+    benchmark_cmd="bash run_benchmark.sh --topology=${model} --mode=performance --batch_size=1 --iters=500"
 elif [ "${model}" == "resnet-101" ]; then
     model_src_dir="oob_models/quantization/ptq"
     dataset_location=""
@@ -103,8 +106,9 @@ elif [ "${model}" == "resnet-101" ]; then
     strategy="basic"
     batch_size=1
     new_benchmark=false
-    tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}"
-    benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500"
+    inc_new_api=true
+    tuning_cmd="bash run_tuning.sh --topology=${model} --input_model=${input_model}"
+    benchmark_cmd="bash run_benchmark.sh --topology=${model} --mode=performance --batch_size=1 --iters=500"
 elif [ "${model}" == "resnet50_fashion" ]; then
     model_src_dir="image_recognition/keras_models/resnet50_fashion/quantization/ptq"
     dataset_location="/tf_dataset2/datasets/mnist/FashionMNIST_small"
@@ -131,6 +135,7 @@ fi
     --new_benchmark=${new_benchmark} \
     --tuning_cmd="${tuning_cmd}" \
     --benchmark_cmd="${benchmark_cmd}" \
+    --inc_new_api="${inc_new_api}" \
     --mode=${mode} \
     --USE_TUNE_ACC=${USE_TUNE_ACC} \
     --PERF_STABLE_CHECK=${PERF_STABLE_CHECK} \