Perf stable check

.azure-pipelines/model-test.yml

@@ -47,13 +47,12 @@ parameters:
     type: object
     default:
       - resnet50-v1-12
-      - bert_base_MRPC_dynamic
+      # - bert_base_MRPC_dynamic
   - name: MXNetModelList
     type: object
     default:
       - resnet50v1
 
-
 stages:
   - stage: TensorFlowModels
     displayName: Run TensorFlow Model
@@ -111,7 +110,6 @@ stages:
                   modelName: ${{ model }}
                   framework: "mxnet"
 
-
   - stage: GenerateLogs
     displayName: Generate Report
     dependsOn: [TensorFlowModels, PyTorchModels, MXNetModels, ONNXModels]
@@ -138,9 +136,9 @@ stages:
               patterns: "**/*_tuning_info.log"
               path: $(OUT_SCRIPT_PATH)
           - task: UsePythonVersion@0
-            displayName: "Use Python 3."
+            displayName: "Use Python 3.8"
             inputs:
-              versionSpec: "3"
+              versionSpec: "3.8"
           - script: |
               cd ${OUT_SCRIPT_PATH}
               mkdir generated
@@ -154,15 +152,15 @@ stages:
               patterns: "**.log"
               path: $(OUT_SCRIPT_PATH)/last_generated
               project: $(System.TeamProject)
-              pipeline: "Baseline"
+              pipeline: "Model-Test"
               runVersion: "specific"
-              runId: $(model_runID)
+              runId: $(refer_buildId)
               retryDownloadCount: 3
             displayName: "Download last logs"
           - script: |
               echo "------ Generating final report.html ------"
               cd ${OUT_SCRIPT_PATH}
-              /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated --ghprbPullId $(System.PullRequest.PullRequestNumber) --MR_source_branch $(System.PullRequest.SourceBranch) --MR_source_repo $(System.PullRequest.SourceRepositoryURI) --MR_target_branch $(System.PullRequest.TargetBranch) --repo_url $(Build.Repository.Uri) --source_commit_id $(Build.SourceVersion) --build_id $(Build.BuildId)
+              /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated
             displayName: "Generate report"
           - task: PublishPipelineArtifact@1
             inputs:

.azure-pipelines/scripts/install_nc.sh

@@ -4,4 +4,4 @@ cd /neural-compressor
 python -m pip install --no-cache-dir -r requirements.txt
 python setup.py sdist bdist_wheel
 pip install dist/neural_compressor*.whl
-pip list
+pip list

.azure-pipelines/scripts/install_nc_full.sh

@@ -4,4 +4,4 @@ cd /neural-compressor
 python -m pip install --no-cache-dir -r requirements.txt
 python setup.py --full sdist bdist_wheel
 pip install dist/neural_compressor*.whl
-pip list
+pip list

.azure-pipelines/scripts/models/collect_log_all.py

@@ -1,7 +1,5 @@
-import re
-import os
-import platform
 import argparse
+import os
 
 parser = argparse.ArgumentParser(allow_abbrev=False)
 parser.add_argument("--logs_dir", type=str, default=".")

.azure-pipelines/scripts/models/collect_log_model.py

@@ -1,8 +1,6 @@
-import re
-import os
-import platform
 import argparse
-
+import os
+import re
 
 parser = argparse.ArgumentParser(allow_abbrev=False)
 parser.add_argument("--framework", type=str, required=True)
@@ -11,20 +9,127 @@
 parser.add_argument("--logs_dir", type=str, default=".")
 parser.add_argument("--output_dir", type=str, default=".")
 parser.add_argument("--build_id", type=str, default="3117")
+parser.add_argument("--stage", type=str, default="collect_log")
 args = parser.parse_args()
 print('===== collecting log model =======')
 print('build_id: '+args.build_id)
 OS='linux'
 PLATFORM='icx'
 URL ='https://dev.azure.com/lpot-inc/neural-compressor/_build/results?buildId='+args.build_id+'&view=artifacts&pathAsName=false&type=publishedArtifacts'
 
-print(args)
+
+def get_model_tuning_dict_results():
+    tuning_result_dict = {}
+
+    if os.path.exists(tuning_log):
+        print('tuning log found')
+        tmp = {'fp32_acc': 0, 'int8_acc': 0, 'tuning_trials': 0}
+        with open(tuning_log, "r") as f:
+            for line in f:
+                parse_tuning_line(line, tmp)
+        print(tmp)
+        # set model status failed
+        if tmp['fp32_acc'] == 0 or tmp['int8_acc'] == 0:
+            os.system('echo "##vso[task.setvariable variable=' + args.framework + '_' + args.model + '_failed]true"')
+
+        tuning_result_dict = {
+            "OS": OS,
+            "Platform": PLATFORM,
+            "Framework": args.framework,
+            "Version": args.fwk_ver,
+            "Model": args.model,
+            "Strategy": tmp['strategy'],
+            "Tune_time": tmp['tune_time'],
+        }
+        benchmark_accuracy_result_dict = {
+            'int8': {
+                "OS": OS,
+                "Platform": PLATFORM,
+                "Framework": args.framework,
+                "Version": args.fwk_ver,
+                "Model": args.model,
+                "Mode": "Inference",
+                "Type": "Accuracy",
+                "BS": 1,
+                "Value": tmp['int8_acc'],
+                "Url": URL,
+            },
+            'fp32': {
+                "OS": OS,
+                "Platform": PLATFORM,
+                "Framework": args.framework,
+                "Version": args.fwk_ver,
+                "Model": args.model,
+                "Mode": "Inference",
+                "Type": "Accuracy",
+                "BS": 1,
+                "Value": tmp['fp32_acc'],
+                "Url": URL,
+            }
+        }
+
+        return tuning_result_dict, benchmark_accuracy_result_dict
+    else:
+        return {}, {}
+
+
+def get_model_benchmark_dict_results():
+    benchmark_performance_result_dict = {"int8": {}, "fp32": {}}
+    for precision in ["int8", "fp32"]:
+        throughput = 0.0
+        bs = 1
+        for root, dirs, files in os.walk(args.logs_dir):
+            for name in files:
+                file_name = os.path.join(root, name)
+                print(file_name)
+                if "performance-" + precision in name:
+                    for line in open(file_name, "r"):
+                        result = parse_perf_line(line)
+                        if result.get("throughput"):
+                            throughput += result.get("throughput")
+                        if result.get("batch_size"):
+                            bs = result.get("batch_size")
+
+        # set model status failed
+        if throughput == 0.0:
+            os.system('echo "##vso[task.setvariable variable=' + args.framework + '_' + args.model + '_failed]true"')
+        benchmark_performance_result_dict[precision] = {
+            "OS": OS,
+            "Platform": PLATFORM,
+            "Framework": args.framework,
+            "Version": args.fwk_ver,
+            "Model": args.model,
+            "Mode": "Inference",
+            "Type": "Performance",
+            "BS": 1,
+            "Value": throughput,
+            "Url": URL,
+        }
+
+    return benchmark_performance_result_dict
+
+
+def get_refer_data():
+    refer_log = os.path.join(f"{args.logs_dir}_refer_log", f"{args.framework}_{args.model}_summary.log")
+    result = {}
+    if os.path.exists(refer_log):
+        with open(refer_log, "r") as f:
+            lines = f.readlines()
+            keys = lines[0].split(";")
+            values = [lines[i].split(";") for i in range(1, len(lines))]
+        for value in values:
+            precision = value[keys.index("Precision")]
+            Type = value[keys.index("Type")]
+            result[f"{precision}_{Type}"] = float(value[keys.index("Value")])
+        return result
+    else:
+        print(f"refer log file: {refer_log} not found")
+        return 0
 
 
-def main():
+def collect_log():
     results = []
     tuning_infos = []
-    tuning_log = os.path.join(args.logs_dir, f"{args.framework}-{args.model}-tune.log")
     print("tuning log dir is {}".format(tuning_log))
     # get model tuning results
     if os.path.exists(tuning_log):
@@ -123,5 +228,29 @@ def parse_perf_line(line) -> float:
     return perf_data
 
 
+def check_status(precision, precision_upper, check_accuracy = False):
+    performance_result = get_model_benchmark_dict_results()
+    current_performance = performance_result.get(precision).get("Value")
+    refer_performance = refer.get(f"{precision_upper}_Performance")
+    print(f"current_performance_data = {current_performance}, refer_performance_data = {refer_performance}")
+    assert abs(current_performance - refer_performance) / refer_performance <= 0.05
+
+    if check_accuracy:
+        _, accuracy_result = get_model_tuning_dict_results()
+        current_accuracy = accuracy_result.get(precision).get("Value")
+        refer_accuracy = refer.get(f"{precision_upper}_Accuracy")
+        print(f"current_accuracy_data = {current_accuracy}, refer_accuarcy_data = {refer_accuracy}")
+        assert abs(current_accuracy - refer_accuracy) / refer_accuracy <= 0.05
+
+
 if __name__ == '__main__':
-    main()
+    tuning_log = os.path.join(args.logs_dir, f"{args.framework}-{args.model}-tune.log")
+    refer = get_refer_data()
+    if args.stage == "collect_log":
+        collect_log()
+    elif args.stage == "int8_benchmark":
+        check_status("int8", "INT8")
+    elif args.stage == "fp32_benchmark":
+        check_status("fp32", "FP32")
+    else:
+        raise ValueError(f"{args.stage} does not exist")

.azure-pipelines/scripts/models/env_setup.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+set -eo pipefail
+source /neural-compressor/.azure-pipelines/scripts/change_color.sh
+
+# get parameters
+PATTERN='[-a-zA-Z0-9_]*='
+
+for i in "$@"; do
+    case $i in
+    --yaml=*)
+        yaml=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --framework=*)
+        framework=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --fwk_ver=*)
+        fwk_ver=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --torch_vision_ver=*)
+        torch_vision_ver=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --model=*)
+        model=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --model_src_dir=*)
+        model_src_dir=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --dataset_location=*)
+        dataset_location=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --batch_size=*)
+        batch_size=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --strategy=*)
+        strategy=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    --new_benchmark=*)
+        new_benchmark=$(echo $i | sed "s/${PATTERN}//")
+        ;;
+    *)
+        echo "Parameter $i not recognized."
+        exit 1
+        ;;
+    esac
+done
+
+SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models"
+log_dir="/neural-compressor/.azure-pipelines/scripts/models"
+WORK_SOURCE_DIR="/neural-compressor/examples/${framework}"
+$BOLD_YELLOW && echo "processing ${framework}-${fwk_ver}-${model}" && $RESET
+
+$BOLD_YELLOW && echo "======= creat log_dir =========" && $RESET
+if [ -d "${log_dir}/${model}" ]; then
+    $BOLD_GREEN && echo "${log_dir}/${model} already exists, don't need to mkdir." && $RESET
+else
+    $BOLD_GREEN && echo "no log dir ${log_dir}/${model}, create." && $RESET
+    cd ${log_dir}
+    mkdir ${model}
+fi
+
+$BOLD_YELLOW && echo "====== install requirements ======" && $RESET
+/bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh
+
+cd ${WORK_SOURCE_DIR}/${model_src_dir}
+pip install ruamel_yaml
+pip install psutil
+pip install protobuf==3.20.1
+if [[ "${framework}" == "tensorflow" ]]; then
+    pip install intel-tensorflow==${fwk_ver}
+elif [[ "${framework}" == "pytorch" ]]; then
+    pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html
+    pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html
+elif [[ "${framework}" == "onnxrt" ]]; then
+    pip install onnx==1.11.0
+    pip install onnxruntime==${fwk_ver}
+elif [[ "${framework}" == "mxnet" ]]; then
+    if [[ "${fwk_ver}" == "1.7.0" ]]; then
+        pip install mxnet==${fwk_ver}.post2
+    elif [[ "${fwk_ver}" == "1.6.0" ]]; then
+        pip install mxnet-mkl==${mxnet_version}
+    else
+        pip install mxnet==${fwk_ver}
+    fi
+fi
+
+if [ -f "requirements.txt" ]; then
+    sed -i '/neural-compressor/d' requirements.txt
+    if [ "${framework}" == "onnxrt" ]; then
+        sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt
+    fi
+    if [ "${framework}" == "tensorflow" ]; then
+        sed -i '/tensorflow==/d;/tensorflow$/d' requirements.txt
+        sed -i '/^intel-tensorflow/d' requirements.txt
+    fi
+    if [ "${framework}" == "mxnet" ]; then
+        sed -i '/mxnet==/d;/mxnet$/d;/mxnet-mkl==/d;/mxnet-mkl$/d' requirements.txt
+    fi
+    if [ "${framework}" == "pytorch" ]; then
+        sed -i '/torch==/d;/torch$/d;/torchvision==/d;/torchvision$/d' requirements.txt
+    fi
+    n=0
+    until [ "$n" -ge 5 ]; do
+        python -m pip install -r requirements.txt && break
+        n=$((n + 1))
+        sleep 5
+    done
+    pip list
+else
+    $BOLD_RED && echo "Not found requirements.txt file." && $RESET
+fi
+
+$BOLD_YELLOW && echo "======== update yaml config ========" && $RESET
+$BOLD_YELLOW && echo -e "\nPrint origin yaml..." && $RESET
+cat ${yaml}
+python ${SCRIPTS_PATH}/update_yaml_config.py \
+    --yaml=${yaml} \
+    --framework=${framework} \
+    --dataset_location=${dataset_location} \
+    --batch_size=${batch_size} \
+    --strategy=${strategy} \
+    --new_benchmark=${new_benchmark} \
+    --multi_instance='true'
+$BOLD_YELLOW && echo -e "\nPrint updated yaml... " && $RESET
+cat ${yaml}