diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml new file mode 100644 index 00000000000..8c975336a15 --- /dev/null +++ b/.azure-pipelines/code-scan.yml @@ -0,0 +1,96 @@ +trigger: none + +pr: + - master + +pool: ICX-16C + +variables: + CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" + +stages: + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + steps: + - template: template/code-scan-template.yml + parameters: + codeScanFileName: 'bandit' + uploadPath: 'lpot-bandit.log' + + - stage: PylintCodeScan + displayName: Pylint Code Scan + dependsOn: [] + jobs: + - job: Pylint + displayName: Pylint + steps: + - template: template/code-scan-template.yml + parameters: + codeScanFileName: 'pylint' + uploadPath: 'lpot-pylint.json' + + - stage: PyspellingCodeScan + displayName: Pyspelling Code Scan + dependsOn: [] + jobs: + - job: Pyspelling + displayName: Pyspelling + steps: + - template: template/code-scan-template.yml + parameters: + codeScanFileName: 'pyspelling' + uploadPath: 'lpot_pyspelling.log' + + - stage: CopyRight + displayName: CopyRight Code Scan + dependsOn: [] + jobs: + - job: CopyRight + displayName: CopyRight + steps: + - script: | + sudo rm -fr $(Build.SourcesDirectory) || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - task: Bash@3 + inputs: + targetType: "inline" + script: | + source $(Build.SourcesDirectory)/.azure-pipelines/scripts/change_color.sh + set -e + mkdir -p $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH) + RESET="echo -en \\E[0m \\n" # close + + supported_extensions=(py, sh, yaml) + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(Build.SourcesDirectory)/neural_compressor > $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/diff.log + files=$(cat $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + + for file in ${files} + do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + $BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" + cat $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET; exit 1 + fi + else + $LIGHT_PURPLE && echo "Skipping ${file}" && $RESET + fi + done + displayName: "CopyRight Check" + + - task: PublishPipelineArtifact@1 + condition: failed() + inputs: + targetPath: $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + artifact: copyright + publishLocation: "pipeline" + displayName: "PublishPipelineArtifact" \ No newline at end of file diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel new file mode 100644 index 00000000000..71ab8bfda6a --- /dev/null +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -0,0 +1,44 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBUNTU_VER=20.04 +FROM ubuntu:${UBUNTU_VER} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + autoconf \ + build-essential \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 \ + numactl \ + time \ + wget \ + vim + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip --no-cache-dir install --upgrade pip +RUN python -m pip install --no-cache-dir setuptools + +RUN pip list + +WORKDIR / + diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.devel new file mode 100644 index 00000000000..93321aa0f14 --- /dev/null +++ b/.azure-pipelines/docker/DockerfileCodeScan.devel @@ -0,0 +1,43 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG IMAGE_NAME='neural-compressor' +ARG IMAGE_TAG='py38' + +FROM ${IMAGE_NAME}:${IMAGE_TAG} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + aspell \ + aspell-en + +RUN python -m pip install --no-cache-dir pylint==2.12.1\ + bandit\ + pyspelling\ + google\ + autograd\ + ofa\ + fvcore\ + pymoo\ + onnxruntime_extensions\ + tf_slim\ + transformers\ + horovod\ + flask==2.1.3 + +WORKDIR / diff --git a/.azure-pipelines/docker/DockerfileWithNC.devel b/.azure-pipelines/docker/DockerfileWithNC.devel new file mode 100644 index 00000000000..113fdead9b9 --- /dev/null +++ b/.azure-pipelines/docker/DockerfileWithNC.devel @@ -0,0 +1,53 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG UBUNTU_VER=20.04 +FROM ubuntu:${UBUNTU_VER} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +ARG REPO=x +ARG TARGET_BRANCH=y +ARG SOURCE_BRANCH=z +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + autoconf \ + build-essential \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip --no-cache-dir install --upgrade pip + +RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ + cd neural-compressor && \ + python -m pip install --no-cache-dir setuptools && \ + python -m pip install --no-cache-dir pycocotools && \ + python -m pip install --no-cache-dir -r requirements.txt && \ + python setup.py install + +RUN pip list +RUN pip list | grep neural + +WORKDIR /neural-compressor +RUN git rev-parse HEAD \ No newline at end of file diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml new file mode 100644 index 00000000000..c63d03cc6a4 --- /dev/null +++ b/.azure-pipelines/model-test.yml @@ -0,0 +1,176 @@ +trigger: none + +pr: + - master + +pool: ICX-16C + +variables: + OUT_SCRIPT_PATH: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models + SCRIPT_PATH: /neural-compressor/.azure-pipelines/scripts + +parameters: + - name: TensorFlow_Model + displayName: Run TensorFlow models? + type: boolean + default: true + - name: PyTorch_Model + displayName: Run PyTorch models? + type: boolean + default: true + - name: ONNX_Model + displayName: Run ONNX models? + type: boolean + default: true + - name: MXNet_Model + displayName: Run MXNet models? + type: boolean + default: true + + - name: TensorFlowModelList + type: object + default: + - resnet50v1.5 + - ssd_resnet50_v1 + - ssd_mobilenet_v1_ckpt + - inception_v1 + - resnet50_fashion + - darknet19 + - densenet-121 + - resnet-101 + - name: PyTorchModelList + type: object + default: + - resnet18 + - resnet18_fx + - name: ONNXModelList + type: object + default: + - resnet50-v1-12 + # - bert_base_MRPC_dynamic + - name: MXNetModelList + type: object + default: + - resnet50v1 + +stages: + - stage: TensorFlowModels + displayName: Run TensorFlow Model + dependsOn: [] + condition: eq('${{ parameters.TensorFlow_Model }}', 'true') + jobs: + - ${{ each model in parameters.TensorFlowModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "tensorflow" + + - stage: PyTorchModels + displayName: Run PyTorch Model + dependsOn: [] + condition: eq('${{ parameters.PyTorch_Model }}', 'true') + jobs: + - ${{ each model in parameters.PyTorchModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "pytorch" + + - stage: ONNXModels + displayName: Run ONNX Model + dependsOn: [] + condition: eq('${{ parameters.ONNX_Model }}', 'true') + jobs: + - ${{ each model in parameters.ONNXModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "onnxrt" + + - stage: MXNetModels + displayName: Run MXNet Model + dependsOn: [] + condition: eq('${{ parameters.MXNet_Model }}', 'true') + jobs: + - ${{ each model in parameters.MXNetModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "mxnet" + + - stage: GenerateLogs + displayName: Generate Report + dependsOn: [TensorFlowModels, PyTorchModels, MXNetModels, ONNXModels] + condition: always() + jobs: + - job: GenerateReport + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + clean: true + displayName: "Checkout out Repo" + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + patterns: "**/*_summary.log" + path: $(OUT_SCRIPT_PATH) + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + patterns: "**/*_tuning_info.log" + path: $(OUT_SCRIPT_PATH) + - task: UsePythonVersion@0 + displayName: "Use Python 3.8" + inputs: + versionSpec: "3.8" + - script: | + cd ${OUT_SCRIPT_PATH} + mkdir generated + mkdir last_generated + python -u collect_log_all.py --logs_dir $(OUT_SCRIPT_PATH) --output_dir generated + displayName: "Collect all logs" + - task: DownloadPipelineArtifact@2 + inputs: + source: "specific" + artifact: "FinalReport" + patterns: "**.log" + path: $(OUT_SCRIPT_PATH)/last_generated + project: $(System.TeamProject) + pipeline: "Model-Test" + runVersion: "specific" + runId: $(refer_buildId) + retryDownloadCount: 3 + displayName: "Download last logs" + - script: | + echo "------ Generating final report.html ------" + cd ${OUT_SCRIPT_PATH} + /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated + displayName: "Generate report" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(OUT_SCRIPT_PATH)/generated + artifact: FinalReport + publishLocation: "pipeline" + displayName: "Publish report" + - script: | + if [ $(is_perf_reg) == 'true' ]; then + echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports." + exit 1 + fi + displayName: "Specify performance regression" diff --git a/.azure-pipelines/scripts/change_color.sh b/.azure-pipelines/scripts/change_color.sh new file mode 100644 index 00000000000..8cbd611cba4 --- /dev/null +++ b/.azure-pipelines/scripts/change_color.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# -------------- general approach start---------------- + +# 1. import this file: + # source path/change_color.sh +# 2. use COLOR/BG: + # $VARIABLE_NAME && out_put_content && $RESET +# 3. COLOR + BG: + # $COLOR/BG_VARIABLE_NAME && $BG/COLOR_VARIABLE_NAME && out_put_content && $RESET +# 4. custom + # abbreviation(change number) + # txt number range (30, 37) + # bg number range (40, 47) + # special effects number range (1, 7) + # echo -en \\E[number1 + ; + number2 + ; + number3 + m" + # e.g - BG_GRAY+LIGHT_RED = "echo -en \\E[47;31m" + +# -------------- general approach end----------------== + + +# general setting +# ------------- light_color start---------------- +# black +LIGHT_BLACK="echo -en \\E[30m" +# red +LIGHT_RED="echo -en \\E[31m" +# green +LIGHT_GREEN="echo -en \\E[32m" +# yellow +LIGHT_YELLOW="echo -en \\E[33m" +# blue +LIGHT_BLUE="echo -en \\E[34m" +# purple +LIGHT_PURPLE="echo -en \\E[35m" +# cyan +LIGHT_CYAN="echo -en \\E[36m" +# gray +LIGHT_GRAY="echo -en \\E[37m" +# ------------- light_color end---------------- + +# ------------- bold_color start---------------- +# black +BOLD_BLACK="echo -en \\E[1;30m" +# red +BOLD_RED="echo -en \\E[1;31m" +# green +BOLD_GREEN="echo -en \\E[1;32m" +# yellow +BOLD_YELLOW="echo -en \\E[1;33m" +# blue +BOLD_BLUE="echo -en \\E[1;34m" +# purple +BOLD_PURPLE="echo -en \\E[1;35m" +# cyan +BOLD_CYAN="echo -en \\E[1;36m" +# gray +BOLD_GRAY="echo -en \\E[1;37m" +# ------------- bold_color end---------------- + +# ------------- background_color start---------------- +# black +BG_BLACK="echo -en \\E[40m" +# red +BG_RED="echo -en \\E[41m" +# green +BG_GREEN="echo -en \\E[42m" +# yellow +BG_YELLOW="echo -en \\E[43m" +# blue +BG_BLUE="echo -en \\E[44m" +# purple +BG_PURPLE="echo -en \\E[45m" +# cyan +BG_CYAN="echo -en \\E[46m" +# gray +BG_GRAY="echo -en \\E[47m" +# ------------- background_color end---------------- + +# close +RESET="echo -en \\E[0m" diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh new file mode 100644 index 00000000000..a23f2f3000d --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +source /neural-compressor/.azure-pipelines/scripts/change_color.sh +mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog +bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" + +python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log +exit_code=$? + +# code-scan close +RESET="echo -en \\E[0m \\n" + +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" +cat $bandit_log_dir/lpot-bandit.log +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + + +if [ ${exit_code} -ne 0 ] ; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details." && $RESET; exit 1 +fi +$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh new file mode 100644 index 00000000000..a1d85f305ae --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +source /neural-compressor/.azure-pipelines/scripts/change_color.sh +mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog +pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" + +pip install -r /neural-compressor/requirements.txt + +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json +exit_code=$? + +# code-scan close +RESET="echo -en \\E[0m \\n" + +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" +cat $pylint_log_dir/lpot-pylint.json +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + + +if [ ${exit_code} -ne 0 ] ; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET; exit 1 +fi +$BOLD_PURPLE && echo "Congratulations, Pylint check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt new file mode 100644 index 00000000000..da2499a7954 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -0,0 +1,2277 @@ +aa +aac +abc +Abc +AbcAdaptor +AbcTuneStrategy +abi +ABI +absl +abspath +abstractive +acc +Acc +accuracies +acdc +ACDC +Acknowledgement +activations +Adadelta +adam +AdamW +adaptor +Adaptor +AddEmbeddings +AddN +addr +ADDR +AddV +ade +ADE +adresses +AdvProp +ae +aea +af +AGS +ai +ailab +al +albert +alexnet +AlexNet +algo +algos +alibaba +Alibaba +AlignImageChannel +allenai +alloc +ALLREDUCE +alsologtostderr +Aman +amazonaws +amazonlinux +Amodei +AmpConf +AMX +analytics +Analytics +Anastasiia +AnchorGenerator +andravin +andreamad +anisotropic +anno +anton +ap +apache +api +APIs +APl +APm +approch +APs +arg +argmax +ArgMax +args +arxiv +arXiv +asd +astype +asym +async +atrous +att +AttentionReshape +attr +attredirects +AttributeProto +attrs +auc +aug +autgrad +autogenerate +autograd +AutoMixPrecision +autopep +Autoregressive +ava +AverageMeter +avgloss +AvgPool +avx +AVX +backend +backends +backticks +bart +barthez +bashrc +basicConfig +BasicTokenizer +batchgenerators +BatchMatMul +BatchMatMulV +batchnorm +BatchNorm +bayesian +BayesianOptimization +bazel +bbbb +bbox +bboxes +bccf +bce +bd +bdb +bdist +benchmarked +benchmarking +Benchmarking +Bengio +Benoît +berkeleyvision +bert +BERT's +BertAdam +BertConfig +BERTDataSet +BertForMaskedLM +BertForNextSentencePrediction +BertForPreTraining +BertForQuestionAnswering +BertForSequenceClassification +BertForTokenClassification +BertModel +berts +bertsquad +BertTokenizer +bfloat +BFP +BGR +Bianchi +BiasAdd +BibTeX +bicubic +bilibili +BiliBili +bilinear +BilinearImagenet +billsum +BiLSTM +binarize +binarized +BinaryAdd +biomedical +Biomedical +BLAS +blendcnn +BlendCnn +BlendCNN +bleu +BLEU +blocktime +blogpost +bn +bninception +BNInception +bobw +booktitle +bool +BoW +boxlist +BoxList +br +BrainTumour +BraTS +broadcasted +bs +bsnone +bugfix +buildin +builtin +Builtin +BV +bvlcalexnet +bzip +cadene +Cadene +caffe +Caffe +caffenet +Caffenet +cafferesnet +CaffeResnet +CaffeResNet +Caiming +calib +calibrationcsv +camembert +CamemBERT +canada +Carbonell +CascadeFullRes +cbica +cd +cdn +ce +cec +CenterCrop +centernet +centerNet +centos +CentOS +Centre +cern +CERN's +certfile +Cesa +cfa +cffi +cfg +CFLAGS +ChamNet +Chaumond +checkbox +checkboxes +Cheng +chmod +Cho's +chongruo +Chongruo +chris +Chuanqi +ci +cifar +CIFAR +circleci +Cistac +cityscapes +Cityscapes +cityscapesscripts +cityscapesScripts +cknowledge +ckpt +ckpts +ClassPredictionTower +clcarwin +Clergerie +cli +CLI +clipnorm +clm +CLM +cls +CLX +cly +cmake +CMake +CMAKE +CMakeLists +cmd +CMU +cn +cnn +cnt +CoCo +cocoapi +cocoApi +cocodataset +COCODataset +COCODemo +COCOEval +COCOmAP +COCOmAPv +COCONpy +cocoraw +COCORaw +COCORecord +codalab +codecogs +codenamed +CoLA +colorama +ColorJitter +colspan +compat +compilervars +concat +ConcatV +cond +conda +CONDA +condconv +CondConv +Condensenet +conf +config +Config +configs +CoNLL +Conneau +const +ConstantOfShape +ConstDataLoader +constexpr +contaning +conv +Conv +ConvBNReLU +ConvertingSSDMobilenetToONNX +convertion +ConvNets +convolutional +Convolutional +ConvPerStage +ConvReLU +copt +coreml +CoreML +cp +cpp +cpu +cpus +CPUs +CPX +cpython +creafz +creatis +creativecommons +criteo +Criteo +CriteoTerabyte +croping +CropResize +CropToBoundingBox +CrossEntropyLoss +crossvalidaton +crt +csv +ctrl +CTRLModel +CTRLTokenizer +ctuning +ctx +cuda +cudaPopCallConfiguration +cudatoolkit +CUDAToolKit +cudnn +CUHK +curr +Curran +custormer +cv +CvAClvFfyA +CXX +cxxopt +cypw +cython +da +dae +DagnyT +Dai +dailymail +Danqi +darknet +Darknet +datadir +datafile +dataloader +dataLoader +DataLoader +DataLoadermodule +dataloaders +DataParallel +datapoints +DataProcessor +dataset +Dataset +DatasetAnalyzer +datasets +datatype +datatypes +dathath +Dathathri +datset +dbmdz +DBMDZ +dbox +dbs +DCMAKE +dcn +ddp +DDP +DDR +de +deberta +DecodeImage +deepengine +deeplab +DeepLab +deeplabv +DeepLabV +DeepLearningExamples +Delangue +DENABLE +denseblock +denselayer +densenet +DenseNet +deps +DepthwiseConv +dequant +dequantize +DequantizeLinear +DequantStub +DeQuantStub +desc +dest +destructor +detections +detectron +Detectron +dev +devel +Devlin +devtool +DFabiansResUNet +DFS +DGAN +dialogpt +DialoGPT +dicts +dir +dirname +Discrim +distil +Distil +distilbert +DistilBert +DistilBERT +DistilBERT +DistilBertModel +DistilBertTokenizer +distilgpt +DistilGPT +distillated +Distillated +distillating +DistilmBERT +distilrobert +distilroberta +DistilRoBERTa +DistributedDataParallel +DistributedOptimizer +DistributedSampler +distro +dividiti +Djamé +DKFZ +dl +dlabel +dlboost +dlrm +DLRM's +dmlc +DMQA +dNative +dnf +dnn +dnnl +DNNL +Dockerfile +doclist +docstrings +doctrings +docutils +doteq +dowmsampling +downloader +downsampled +downsampling +doxygen +dpn +DPNs +dpr +DropOut +ds +dscore +dst +dtype +DualPathNet +dualpathnetworks +DualPathNetworks +DummyDataLoader +dunet +DUNet +Dupont +Durand +dvdt +dw +dynamiccaly +ead +EAQkaohzrJbd +earlystop +eb +ecotrust +edgetpu +EdgeTPU +edu +ee +eer +ef +efficientnet +efficientNet +EfficientNet +EfficientNets +eg +eightbit +einstein +el +elif +eltwise +emb +embeddings +embs +EMC +enablerepo +EncodeJped +enfr +eng +ensembling +ensp +entrypoint +enum +env +eq +erf +Erf +Éric +eriklindernoren +Errno +esri +et +eval +evaluator +evel +exemplarily +exising +existing +exmaple +expanduser +ExperimentPlanner +ExperimentPlanners +extractive +EzjbRL +fabian +FabiansUNet +facebook +FaceBook +facebookresearch +fairseq +fallbacks +fanout +faq +Farhadi +FashionMNIST +FasterRCNN +FastFormers +fastrcnn +fatihcakirs +favourably +fb +fbgemm +FBNet +fbnetc +fbresnet +FBResNet +fc +fcn +FCN +fd +FeatureExtractor +feedbacks +Feng +ffc +filename +filenames +FileNotFoundError +filepath +filesystem +finbert +finetune +Finetune +finetuned +finetuning +flac +FlatMapDataset +flaubert +flavour +flavours +Flavours +floatfunctional +FloatFunctional +FloatTensor +FLOPs +Florian +fmfn +fmt +fmtstr +fn +fname +fns +foregound +fp +FP +fpic +fPIC +fpn +FPN +FRN +FromConfig +frontend +fstack +ftfy +Fu +fullres +func +functionalities +functionet +functools +Funtowicz +fw +FWK +fx +GameAI +GANs +Garnett +gcc +gclient +gd +geffnet +gelu +Gelu +GeluOperator +GenEfficientNet +GenericPreprocessor +german +germeval +GermEval +gestaltit +getitem +getsize +GetStrides +GFLOPs +gh +gid +Gimpel +Girshick +github +GitHub +githubusercontent +gitmodules +GLIBCXX +GLOG +GLUE +gluebenchmark +gluepy +gluon +Gluon +gluoncv +GluonCV +gluonnlp +gn +GN +goldsborough +goog +google +googleapis +googleblog +googlenet +googlesource +Goyal +gpg +GPG +gpt +gpu +gpus +GPUs +graphdef +GraphDef +GraphModule +GraphProto +Grauman +grpc +gtFile +gtFine +Gui +Guillaume +Guoming +gz +gzY +Haibin +haibinlin +Haihao +hangzhang +hardcoding +HasAns +hawq +HAWQ +HdQ +heatmaps +Hein +helloworld +HelloWorld +henson +hiddenlayer +hippocampus +Hippocampus +HistogramObserver +hlu +horovod +Horovod +HOROVOD +horovodrun +hostfile +Hounsfield +howpublished +HqEgzS +href +html +http +https +Hu +hubert +huggingface +HuggingFace +HuggingFace's +HuggingFacesTS +hujie +hvd +HybirdBlock +HybridBlock +hyperparameter +hyperparameters +icc +ICCV +Icelake +icpc +icx +ide +idx +ie +IEEE +ILSVR +ilsvrc +ILSVRC +Ilya +im +imagecocodataset +ImageFolder +ImageList +imagenet +ImageNet +ImagenetRaw +ImageRecord +ImageRecordIter +imagesTr +imagesTs +img +imgrec +imgs +imgx +IML +impl +ImportError +IMS +inceptionresnetv +InceptionResNetV +inceptionv +InceptionV +incollection +IndexType +indexValue +indices +indico +inferencer +informations +infos +init +InnerProduct +innersource +inp +inplace +inproceedings +inputcsv +InputData +InputExample +InputFile +Inria +insa +instanceonly +instantiation +integerops +intel +intelai +IntelAI +interoperability +introudces +ints +inturn +InvertedResidual +io +ios +iOS +iou +IoU +ipc +ipex +IPEX +ipynb +ipython +ir +irv +ISA +Isensee +isinstance +issuecomment +IssueQuery +IssueQueryThreads +iter +IteratorGetNext +iters +Jäger +jemalloc +Jens +Jie +jim +Jingfei +Jiong +jit +jitter +Joshi +jpeg +JPEGImages +jpg +jpwarren +json +jsons +Julien +JunWang +jupyter +kaggle +kaggleAdDisplayChallenge +kaiming +KaimingHe +Karthik +kcho +keepbs +keepdim +keras +Keskar +keyfile +keypoint +Keypoint +kimiyoung +kitti +kmp +KMP +KnowledgeDistillationLoss +kriz +kwargs +Kyunghyun +LabelShift +labelsTr +Lample +Lan +lang +LanguageModeling +Lapata +Larochelle +LastLayerShape +latencies +LaTeX +Lavin +layernorm +LayerNorm +layoutlm +ld +len +LessEqual +lf +lfaidata +lfs +li +libdeep +libengine +libffi +libGL +libglib +libiomp +libmlperf +librispeech +LibriSpeech +librosa +libsndfile +libstdc +libz +licence +liKE +Limitting +lin +linkopt +linoxide +linux +linuxfoundation +ListDataset +LiTS +Liu +Liu's +llvmlite +lm +LMHeadModel +ln +loadgen +LoadGen +LOADGEN +LoadGen's +LoadgenAPI +LoadgenAPITestSettings +LoadgenVersion +LoadImage +LOC +localdisk +localhost +LOCderiv +LOCpart +logdir +logfile +login +logits +LOGLEVEL +LogSettings +logtostderr +longformer +lossy +Louf +LowPrecisionInferenceTool +lowproposals +lowres +Lp +lpot +LPOT +LPOT's +lr +lS +LSVRC +lt +LTS +lua +Luan +lutzroeder +lyon +Lysandre +lzma +macOS +MACOSX +MAdds +Madotto +MagnitudePrunePolicy +Maier +mainpage +Makefile +MakefileGnProj +MakeIterator +Mandar +Manmatha +manylinux +mAp +mAP +Mapillary +marianmt +MaskPostProcessor +maskrcnn +MaskRCNN +MaskRCNNFPNFeatureExtractor +maskrnn +massa +Massa +matcher +matmul +MatMul +MatMulWithBias +MatMulWithBiasAdd +MatMulWithBiasGelu +MatMulWithBiasTanh +matplotlib +matricses +maxdepth +maxindrange +maxk +MaxPool +maxSizeInComplete +mbart +mBERT +mcc +McCann +mcordts +md +MeanSquaredError +measurer +Medcial +medicaldecathlon +meetup +mem +membind +mems +messi +metabuild +metadata +metamind +MICCAI +microsoft +miguelgrinberg +Mingda +minibatch +minilm +minimalistic +minival +minloglevel +minmax +MinMaxObserver +mins +mIoU +mIOU +Mirella +misalignments +miscs +Mish +missmatches +MixedConv +mixnet +MixNet +mixup +mkdir +mkl +MKL +mlap +mlas +MLAS +mlcommons +mll +mlm +mlp +mlpc +mlperf +MLperf +MLPerf +mlt +mmdetection +mmlab +MMLAB +mnasnet +MNASNet +mnist +MNIST +mnli +MNLI +mobilebert +MobileBERT +mobilenet +MobileNet +mobilenetv +Mobilenetv +MobileNetv +MobileNetV +modalities +Modalities +modality +ModelConversion +modelfeatures +modelforward +modelinput +modellogits +modelmean +modelsize +modelstd +ModuleDict +ModuleNotFoundError +Molino +mpi +mrcnn +mrpc +MRPC +MSD +mse +MSE +msvc +mul +mult +multi +Multi +multiclass +multilabel +multinli +MultiNLI +multiscale +MULTISCALE +MultiStream +MultiStream's +MultiStreamFree +mutli +mv +mx +mxnet +MxNet +MXNet +MyDataset +Mykhailo +Myle +MyMetric +myModel +MYTASK +MYTASKNAME +Naman +namedtuple +nanohanno +Narasimhan +NAS +nasnet +NASNet +nasnetalarge +nasnetamobile +nb +nbest +nbsp +nc +NCCL +nchw +NCHW +nd +ndarray +NDArray +nderlu +nepoch +ner +NER +nervanasystems +nesterov +NetEase +netron +Netron +networkbuilders +NeurIPS +neval +NewMetric +newstest +nextplatform +ng +ngatang +NGPUS +ngram +NHWC +NIC +nifti +niftis +nii +Nijmegen +Nitish +nl +NLG +nli +nll +nlp +NLP +nlpyang +nltk +NLU +nm +nms +nn +nnodes +nnu +nnU +nnunet +nnUnet +nnUNet +nnUNetPlansv +nnUNetTrainer +nnUNetTrainers +nnUNetTrainerV +NNZ +noduplicates +NoisyStudent +Nonlinearity +NonNestedTuple +NoNormalization +NonZero +noobj +np +nproc +npy +npz +nq +nrix +ns +nsdf +nSsKchNAySU +nthreads +ntrain +num +numactl +numba +numCompleteThreads +numerics +numpy +numTest +numTraining +NVAITC +nvcc +nvidia +NVIDIA +NVIDIA's +nvme +Nx +nyu +ok +ol +Omer +OMP +onboarding +oneapi +oneAPI +onednn +oneDNN +onlinedocs +onnx +ONNX +ONNXQuantizer +onnxrt +ONNXRT +onnxruntime +OnnxRuntime +oob +OOM +OOQtYMH +openai +OpenAI +OpenAI's +OpenAIAdam +OpenAIGPTModel +OpenAIGPTTokenizer +opencv +OpenCV +openmp +openslr +opensource +openssl +openvino +OpenVINO +openvinotoolkit +OpenWebTextCorpus +OperatorConfig +OPs +opset +opsetid +optim +optimizations +Optimizations +optimizers +Optimizers +optypewise +opwise +OrderedDict +ORGderiv +ORGpart +os +osJJ +OTH +OTHderiv +OTHpart +Ott +oup +outdir +OutputData +outputfile +ov +overfeat +overfit +overfitted +PaddingSequence +PaddingSequence +pageId +palletsprojects +panoptic +Panoptic +paperswithcode +param +params +Parinov +ParseDecodeImagenet +ParseDecodeVoc +participations +Parzen +pastebin +patientIDs +pb +pbar +pdf +Peason +pegasus +pelee +peleenet +PeleeNet +Penghui +Pengxin +pepy +PerChannelMinMaxObserver +PERderiv +perf +perftests +PERpart +phrasebank +phy +physcpubind +PhYUmn +Piero +Pierric +PIL +pixAcc +Piyush +pjreddie +pkill +pkl +pky +plm +PLM +pls +pnasnet +PNASNet +png +POC +polynet +PolyNet +Pooler +pos +postprocesing +postprocess +postprocessed +postprocessing +PostProcessor +PostTransform +PowerTools +pplm +PPLM +PQ +pre +prebuild +prebuilt +Prec +precisions +pred +preds +preformance +Preload +preprint +preprocess +preprocessed +preprocesses +preprocessing +preprocessor +Preprocessor +PreprocessorFor +Preprocessors +prerelease +PreSumm +pretrain +pretrained +pretrainedmodels +pretraining +prev +prioritizies +probs +proc +productizing +profilings +ProgressBar +proto +Protobuf +protoc +protractortest +PRs +PrunePolicy +pth +ptq +PTQ +ptr +pudae +pw +PWC +pwd +PWD +px +py +pybind +pycocotools +pyguide +pylint +pymodule +PyObject +pypi +PyPI +PySUT +pytest +PythonAPI +PYTHONPATH +pytorch +PyTorch +pytorchic +PyTorchKnowledgeDistillationLoss +pyyaml +PyYAML +PZ +qat +QAT +qconfig +QConfig +QiaoranC +qint +qlinear +QLinear +qlinearops +QnA +qnli +QNLI +qps +QPS +qqp +QQP +qscheme +qsl +QSL +qtcreator +qtype +quant +quantile +quantizable +Quantizable +quantization +Quantization +quantize +quantized +QuantizedConv +QuantizedConvReLU +QuantizedInput +quantizer +quantizes +Quantizes +quantizing +QuantStub +QueryBackendCapability +QuerySampleComplete +QuerySampleLibrary +quickstart +Quickstart +QuickStart +Quoc +R'emi +Radboud +Radford +Radu +rAjHyXhTzz +rajpurkar +ramdisk +RandAug +RandAugment +randn +RandomCrop +RandomHorizontalFlip +RandomResizedCrop +RandomVerticalFlip +Rault +rc +rcnn +readme +README +ReadmeBuild +ReadmeFAQ +ReadmeHtmlDocs +ReadmeTests +readthedocs +realtime +Realtime +rebase +recommonmark +RecordingObserver +recordio +RecordIO +recurse +Redmon +ReduceMean +regex +RegNet +rehm +Rehm +reinstall +relase +relu +Relu +ReLU +repo +repo's +repo’s +repos +representating +requantize +resampled +resampling +rescale +Rescale +ResencUNet +resize +Resize +ResizeCropImagenet +resized +Resizes +ResizeWithRatio +resnest +ResNest +ResNeSt +resnet +Resnet +ResNet +resnetv +ResNetV +resnext +ResNext +ResNeXt +ressource +ressources +reStructuredText +ret +RetinaMask +retinanet +retinaNet +RetinaNet +reusability +Rewon +rf +rfcn +rgb +RGB +rmax +rmin +RMSE +rn +rng +RNN +rnnt +ro +roberta +RoBERTa +RobertaModel +RobertaTokenizer +ROC +RocStories +Romary +rosanneliu +rougeL +rougeLsum +rowanz +rowspan +RPN +RPNHead +RPNPostProcessor +Rsqrt +rst +rtd +RTX +runhooks +runtime +Runtime +RuntimeError +Rusia +rusiaaman +Ruslan +rw +rwightman +sacremoses +Sagot +Salakhutdinov +salesforce +Salesforce +Salimans +sanh +Sanh +sata +SavedModel +SavedModel +Scalable +scaler +scatterFillKernel +sched +scikit +scm +screenshots +ScriptModule +se +sed +Seddah +seg +segm +SegmentationMask +segmentations +seid +senet +SENet +sentencepiece +Sep +SEP +SeqDataCollator +serializable +ServerPool +sess +setuptools +sexualized +SGD +sgmoid +SHA +sharded +Sharma +Shen +Shirish +shouldn +showEvent +shufflenet +Shufflenet +ShuffleNet +shufflenetv +Shvets +sigmoid +signup +sigopt +Sigopt +SigOpt +SingleStream +skx +Skylake +skylion +SMBO +SMBOs +Smola +smoothes +sndfile +Socher +socio +SocketIO +softmax +somain +Soricut +sota +SOTA +sox +SoX +spacings +spacy +SpaCy +SparseCategoricalAccuracy +SparseCategoricalCrossentropy +sparsified +Spearman +spearmanr +specificities +splitted +spm +spnasnet +sqlalchemy +Sqrt +sqSiUy +Squad +SQuAD +SquadF +squadpy +SquadV +SquaredDifference +squeezebert +squeezenet +SqueezeNet +src +SrcTuple +sryqufw +ssd +SSDMobilenet +SSDSC +sshleifer +sst +stackoverflow +Standley +startswith +StartTest +stdout +stds +stefan +stemblock +stepsize +Stoyanov +str +strided +struct +sts +STS +stsb +styleguide +Suárez +subexpression +subfolder +subfolders +Subgraph +submodule +submodules +Submodules +subsample +subtoken +sudo +Sumanth +summarization +Summarization +SummaryWriter +superseeds +suported +sut +SUT +Sutskever +sv +svg +swagaf +sym +symlink +symlinked +symlinks +Symlinks +synset +sys +SystemUnderTest +tanh +TaskXX +TaskXXX +tb +TBD +tbe +tbody +td +techdecoded +tencent +tensor's +tensorboard +tensorBoard +TensorBoard +tensorcore +TensorDataset +tensorflow +TensorFlow +TensorflowQuery +tensorImageSize +TensorInfo +TensorProto +teraoperations +tesla +testability +TestSettings +tf +TF +TFBertForSequenceClassification +tflite +tfp +tfrecord +TFRecord +TFRecordDataset +tfrecords +TFRobertaModel +TFSlimNetsFactory +TFSlimNetsFactory's +tg +tgt +tgz +th +THCudaTensor +thead +thepath +thres +thrs +Tian +Tidx +timeline +timestamps +TinyBERT +tl +tlkh +tLoss +TLS +tmp +tmpfs +ToArray +ToBGR +toc +toctree +TODO +tokenization +tokenize +tokenized +tokenizer +Tokenizer +tokenizers +Tokenizers +tokenizing +tol +TOL +tolist +toml +ToNDArray +toolchains +ToPILImage +topk +TopK +topologies +ToRange +torchaudio +torchscript +TorchScript +torchtext +torchvision +TorchVision +toronto +totalizing +ToTensor +Toutanova +tp +tpe +TPE +tpu +TPU +tqdm +traceback +trainings +trainval +trainvaltest +transfo +TransformImage +TransfoXLModel +TransfoXLTokenizer +travis +trigram +tstandley +tsv +TuneStrategy +tunings +tuningusage +tuple +tuples +txt +TZ +uber +ubuntu +ubyte +UI +UID +uint +uk +un +uncomment +uncompress +unet +Unet +UNet +unidecode +uniq +unittest +unref +unsqueeze +unstack +upenn +uploader +upscaled +Upscaled +upstreamed +url +userspace +usp +usr +UTC +util +utils +ux +UX +valminusminival +valset +ValueError +Varshney +VCVTNE +VCVTNEPS +VDPBF +vec +Veronika +veronikayurchuk +versioned +Veselin +vgg +viewpage +Villemonte +ViT +voc +VOC +VOCdevkit +VOCmAP +VOCMApMetrics +VOCRecord +voxel +voxels +vram +VRAM +VTune +waleedka +Wallach +wangg +warmup +wav +wd +webcam +Webcam +webite +webpage +WebSockets +WebText +wedam +WeightSharedConvolutionalBoxPredictor +Wformat +wget +whitelist +whl +WideResNet +WideResNet +Wightman +wikipedia +wikitext +WikiText +WilsonCity +WIP +WLYDCRB +wmt +wnd +WnD +wnli +Wnxu +WordPiece +workdir +workflow +Workflow +workflows +workspace +wrt +wwm +www +xad +xception +Xception +xcode +xeon +Xeon +Xiang +Xiong +xl +XLA +xlm +XLMModel +XLMTokenizer +xlnet +XLNet +XLNetModel +XLNetTokenizer +XlUH +xml +xnli +XNLI +xsum +xV +xvf +xvzf +XXXX +xxy +xxz +xYNrZdEAnrHk +xywh +xyxy +xz +xzvf +yacs +yaml +yamls +Yi +Yiming +Yinhan +yizhu +yjxiong +YKd +Yoann +yolo +yolov +YOLOv +YOLOV +yosinski +Yosinski +YqgzY +Yuanjun +Yue +Yunpeng +Yurchuk +YY +zenodo +Zettlemoyer +zfnet +ZFNet +zh +zhang +Zhang +zhanghang +Zhenzhong +Zhi +Zhilin +Zhongyue +zhongyuezhang +Zhu +Zihang +zihangdai +znoexecstack +znow +Zptls +zrelro +zrl +zxvf +CustomObj +ModelSize +QDQ +QLinearOps +qdq +qdqops +CodeGenerator +GEMM +SparseLib +Xbyak +brgemm +cfgs +gtests +hpp +hypotype +kd +ker +kern +sparsednn +spmm +xxxx +GraphModules +wsl +descs +gtest +IOMP +MALLOC +PRETAINED +SPR +libjemalloc +preload +thp +GCP +gcp +gif +solutionslibrary +geomean +VNNI +Preloading +DUC +duc +leftImg +roc +sklearn +CLA +cla +whitehat +codeofconduct +CYP +SBSTD +xd +samsum +IntelCaffe +baremetal +HWs +IC +KH +NVidia +OC +bolded +sparsification +tensorrt +hardwares +BenchmarkConf +PruningConf +DistillationConf +grey +ModelZoo +mzbert +CaffeNet +FlauBERT +GoogleNet +SqueezeBERT +iz +lvwerra +mBart +oje +za +zk +QIntegerops +QLinearops +criterions +HuBERT +csarron +gpb +howey +huawei +noah +nreimers +pruneofa +textattack +scheduler's +BiDAF +bidaf +FERPlus +ferplus +MixedPrecision +DUnetCNN +calibrationset +ndhwc +ArcFace +arcface +arcfaceresnet +nfolds +RFB +WIDERFACE +shuoyang +ultraface +XKeyboard +lscpu +qpa +vnni +xcb +DevCloud +PyPi +aidevcloud +awk +clx +devcloud +lAtr +nda +ppn +qstat +qsub +qsvr +ruserok +scp +spr +stderr +uXXXXX +QuantConf +SuperBench +autocast +kai +mailto +superbench +yao +Lecun +NLPToolkit +Yan +exdb +lecun +publis +yann +abcadf +bcb +INTRA +WARMUPS +ende +intra +inteltensorflow +AutoINC +autoinc +CNWXA +ZHShareTargetIDMore +utm +youtube +zhihu +zhuanlan +AutoQuant +yottx +yrw +Changelog +CHANGELOG +codegen +feedstock +Galata +galata +Javascript +jestjs +jlpm +ui +jpserver +js +JupyterLab +jupyterlab +JupyterLab's +labextension +labextensions +NodeJS +NPM +npm +pkgs +PWDEBUG +pyproject +Releaser +releaser +sdist +ServerAPP +serverextension +serverIP +username +yarnpkg +BasicNAS +DyNAS +NASBase +NASConfig +Supernet +archs +dynas +evals +mbv +nas +nsga +ofa +pareto +pragma +supernet +Governers +OpenMP +cpufreq +governer +powersave +MarkDown +quantizations +NUMA +bc +cdb +deeac +eaf +IntelON +YagFgODM +eD +oQA +qq +weixin \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh new file mode 100644 index 00000000000..5a472fae374 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +source /neural-compressor/.azure-pipelines/scripts/change_color.sh + +mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog +pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" +pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" + +pip install -r /neural-compressor/requirements.txt + +sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml + +pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log +exit_code=$? + +# code-scan close +RESET="echo -en \\E[0m \\n" + +$BOLD_YELLOW && echo "------------------- Current log file output start --------------------------" +cat $pyspelling_log_dir/lpot_pyspelling.log +$BOLD_YELLOW && echo "------------------- Current log file output end ----------------------------" && $RESET + + +if [ ${exit_code} -ne 0 ] ; then + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pyspelling error details." && $RESET; exit 1 +fi +$BOLD_PURPLE && echo "Congratulations, Pyspelling check passed!" && $LIGHT_PURPLE && echo "You can click on the artifact button to see the log details." && $RESET; exit 0 + diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml new file mode 100644 index 00000000000..c407f846ea2 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml @@ -0,0 +1,11 @@ +matrix: +- name: Markdown + hunspell: + d: en_US.ISO8859-15 + dictionary: + wordlists: + - ${VAL_REPO}/pyspelling/lpot_dict.txt + output: ${VAL_REPO}/pyspelling/lpot_dict.dic + sources: + - ${LPOT_REPO}/neural_coder/**/*.md + - ${LPOT_REPO}/neural_coder/*.md \ No newline at end of file diff --git a/.azure-pipelines/scripts/install_nc.sh b/.azure-pipelines/scripts/install_nc.sh new file mode 100644 index 00000000000..fa2daca5d19 --- /dev/null +++ b/.azure-pipelines/scripts/install_nc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /neural-compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py sdist bdist_wheel +pip install dist/neural_compressor*.whl +pip list diff --git a/.azure-pipelines/scripts/install_nc_full.sh b/.azure-pipelines/scripts/install_nc_full.sh new file mode 100644 index 00000000000..7513baeb254 --- /dev/null +++ b/.azure-pipelines/scripts/install_nc_full.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /neural-compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py --full sdist bdist_wheel +pip install dist/neural_compressor*.whl +pip list diff --git a/.azure-pipelines/scripts/models/collect_log_all.py b/.azure-pipelines/scripts/models/collect_log_all.py new file mode 100644 index 00000000000..fb9db0d6721 --- /dev/null +++ b/.azure-pipelines/scripts/models/collect_log_all.py @@ -0,0 +1,39 @@ +import argparse +import os + +parser = argparse.ArgumentParser(allow_abbrev=False) +parser.add_argument("--logs_dir", type=str, default=".") +parser.add_argument("--output_dir", type=str, default=".") +args = parser.parse_args() +print(args) + + +def main(): + file_dir = args.logs_dir + summary_content = ['OS;Platform;Framework;Version;Precision;Model;Mode;Type;BS;Value;Url\n'] + tuning_info_content = ['OS;Platform;Framework;Version;Model;Strategy;Tune_time\n'] + # get full path of all files + for root, dirs, files in os.walk(file_dir): + for name in files: + file_name = os.path.join(root, name) + print(file_name) + if '_summary.log' in name: + for line in open(file_name, "r"): + # print(line) + if 'linux' in line: + summary_content.append(line) + if '_tuning_info.log' in name: + for line in open(file_name, "r"): + # print(line) + if 'linux' in line: + tuning_info_content.append(line) + f = open(args.output_dir + '/summary.log', "a") + for summary in summary_content: + f.writelines(str(summary)) + f2 = open(args.output_dir + '/tuning_info.log', "a") + for tuning_info in tuning_info_content: + f2.writelines(str(tuning_info)) + + +if __name__ == '__main__': + main() diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py new file mode 100644 index 00000000000..7fbfb55dfac --- /dev/null +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -0,0 +1,256 @@ +import argparse +import os +import re + +parser = argparse.ArgumentParser(allow_abbrev=False) +parser.add_argument("--framework", type=str, required=True) +parser.add_argument("--fwk_ver", type=str, required=True) +parser.add_argument("--model", type=str, required=True) +parser.add_argument("--logs_dir", type=str, default=".") +parser.add_argument("--output_dir", type=str, default=".") +parser.add_argument("--build_id", type=str, default="3117") +parser.add_argument("--stage", type=str, default="collect_log") +args = parser.parse_args() +print('===== collecting log model =======') +print('build_id: '+args.build_id) +OS='linux' +PLATFORM='icx' +URL ='https://dev.azure.com/lpot-inc/neural-compressor/_build/results?buildId='+args.build_id+'&view=artifacts&pathAsName=false&type=publishedArtifacts' + + +def get_model_tuning_dict_results(): + tuning_result_dict = {} + + if os.path.exists(tuning_log): + print('tuning log found') + tmp = {'fp32_acc': 0, 'int8_acc': 0, 'tuning_trials': 0} + with open(tuning_log, "r") as f: + for line in f: + parse_tuning_line(line, tmp) + print(tmp) + # set model status failed + if tmp['fp32_acc'] == 0 or tmp['int8_acc'] == 0: + os.system('echo "##vso[task.setvariable variable=' + args.framework + '_' + args.model + '_failed]true"') + + tuning_result_dict = { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Strategy": tmp['strategy'], + "Tune_time": tmp['tune_time'], + } + benchmark_accuracy_result_dict = { + 'int8': { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Mode": "Inference", + "Type": "Accuracy", + "BS": 1, + "Value": tmp['int8_acc'], + "Url": URL, + }, + 'fp32': { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Mode": "Inference", + "Type": "Accuracy", + "BS": 1, + "Value": tmp['fp32_acc'], + "Url": URL, + } + } + + return tuning_result_dict, benchmark_accuracy_result_dict + else: + return {}, {} + + +def get_model_benchmark_dict_results(): + benchmark_performance_result_dict = {"int8": {}, "fp32": {}} + for precision in ["int8", "fp32"]: + throughput = 0.0 + bs = 1 + for root, dirs, files in os.walk(args.logs_dir): + for name in files: + file_name = os.path.join(root, name) + print(file_name) + if "performance-" + precision in name: + for line in open(file_name, "r"): + result = parse_perf_line(line) + if result.get("throughput"): + throughput += result.get("throughput") + if result.get("batch_size"): + bs = result.get("batch_size") + + # set model status failed + if throughput == 0.0: + os.system('echo "##vso[task.setvariable variable=' + args.framework + '_' + args.model + '_failed]true"') + benchmark_performance_result_dict[precision] = { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Mode": "Inference", + "Type": "Performance", + "BS": 1, + "Value": throughput, + "Url": URL, + } + + return benchmark_performance_result_dict + + +def get_refer_data(): + refer_log = os.path.join(f"{args.logs_dir}_refer_log", f"{args.framework}_{args.model}_summary.log") + result = {} + if os.path.exists(refer_log): + with open(refer_log, "r") as f: + lines = f.readlines() + keys = lines[0].split(";") + values = [lines[i].split(";") for i in range(1, len(lines))] + for value in values: + precision = value[keys.index("Precision")] + Type = value[keys.index("Type")] + result[f"{precision}_{Type}"] = float(value[keys.index("Value")]) + return result + else: + print(f"refer log file: {refer_log} not found") + return 0 + + +def collect_log(): + results = [] + tuning_infos = [] + print("tuning log dir is {}".format(tuning_log)) + # get model tuning results + if os.path.exists(tuning_log): + print('tuning log found') + tmp = {'fp32_acc': 0, 'int8_acc': 0, 'tuning_trials': 0} + with open(tuning_log, "r") as f: + for line in f: + parse_tuning_line(line, tmp) + print(tmp) + # set model status failed + if tmp['fp32_acc']==0 or tmp['int8_acc']==0: + os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') + results.append('{};{};{};{};FP32;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['fp32_acc'], URL)) + results.append('{};{};{};{};INT8;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['int8_acc'], URL)) + tuning_infos.append(';'.join([OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['strategy'], str(tmp['tune_time']), str(tmp['tuning_trials']), URL, f"{round(tmp['max_mem_size'] / tmp['total_mem_size'] * 100, 4)}%"])+'\n') + # get model benchmark results + for precision in ['int8', 'fp32']: + throughput = 0.0 + bs = 1 + for root, dirs, files in os.walk(args.logs_dir): + for name in files: + file_name = os.path.join(root, name) + print(file_name) + if 'performance-'+precision in name: + for line in open(file_name, "r"): + result= parse_perf_line(line) + if result.get("throughput"): + throughput += result.get("throughput") + if result.get("batch_size"): + bs = result.get("batch_size") + # set model status failed + if throughput==0.0: + os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') + results.append('{};{};{};{};{};{};Inference;Performance;{};{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, precision.upper(), args.model, bs, throughput, URL)) + # write model logs + f = open(args.output_dir+'/'+args.framework+'_'+args.model+'_summary.log', "a") + f.writelines("OS;Platform;Framework;Version;Precision;Model;Mode;Type;BS;Value;Url\n") + for result in results: + f.writelines(str(result)) + f2 = open(args.output_dir + '/'+args.framework+'_'+args.model+'_tuning_info.log', "a") + f2.writelines("OS;Platform;Framework;Version;Model;Strategy;Tune_time\n") + for tuning_info in tuning_infos: + f2.writelines(str(tuning_info)) + + +def parse_tuning_line(line, tmp): + tuning_strategy = re.search(r"Tuning strategy:\s+([A-Za-z]+)", line) + if tuning_strategy and tuning_strategy.group(1): + tmp['strategy'] = tuning_strategy.group(1) + + baseline_acc = re.search(r"FP32 baseline is:\s+\[Accuracy:\s(\d+(\.\d+)?), Duration \(seconds\):\s*(\d+(\.\d+)?)\]", + line) + if baseline_acc and baseline_acc.group(1): + tmp['fp32_acc'] = float(baseline_acc.group(1)) + + tuned_acc = re.search(r"Best tune result is:\s+\[Accuracy:\s(\d+(\.\d+)?), Duration \(seconds\):\s(\d+(\.\d+)?)\]", line) + if tuned_acc and tuned_acc.group(1): + tmp['int8_acc'] = float(tuned_acc.group(1)) + + tune_trial = re.search(r"Tune \d*\s*result is:", line) + if tune_trial: + tmp['tuning_trials'] += 1 + + tune_time = re.search(r"Tuning time spend:\s+(\d+(\.\d+)?)s", line) + if tune_time and tune_time.group(1): + tmp['tune_time'] = int(tune_time.group(1)) + + fp32_model_size = re.search(r"The input model size is:\s+(\d+(\.\d+)?)", line) + if fp32_model_size and fp32_model_size.group(1): + tmp['fp32_model_size'] = int(fp32_model_size.group(1)) + + int8_model_size = re.search(r"The output model size is:\s+(\d+(\.\d+)?)", line) + if int8_model_size and int8_model_size.group(1): + tmp['int8_model_size'] = int(int8_model_size.group(1)) + + total_mem_size = re.search(r"Total resident size\D*([0-9]+)", line) + if total_mem_size and total_mem_size.group(1): + tmp['total_mem_size'] = float(total_mem_size.group(1)) + + max_mem_size = re.search(r"Maximum resident set size\D*([0-9]+)", line) + if max_mem_size and max_mem_size.group(1): + tmp['max_mem_size'] = float(max_mem_size.group(1)) + + +def parse_perf_line(line) -> float: + perf_data = {} + + throughput = re.search(r"Throughput:\s+(\d+(\.\d+)?)", line) + if throughput and throughput.group(1): + perf_data.update({"throughput": float(throughput.group(1))}) + + batch_size = re.search(r"Batch size = ([0-9]+)", line) + if batch_size and batch_size.group(1): + perf_data.update({"batch_size": int(batch_size.group(1))}) + + return perf_data + + +def check_status(precision, precision_upper, check_accuracy = False): + performance_result = get_model_benchmark_dict_results() + current_performance = performance_result.get(precision).get("Value") + refer_performance = refer.get(f"{precision_upper}_Performance") + print(f"current_performance_data = {current_performance}, refer_performance_data = {refer_performance}") + assert abs(current_performance - refer_performance) / refer_performance <= 0.05 + + if check_accuracy: + _, accuracy_result = get_model_tuning_dict_results() + current_accuracy = accuracy_result.get(precision).get("Value") + refer_accuracy = refer.get(f"{precision_upper}_Accuracy") + print(f"current_accuracy_data = {current_accuracy}, refer_accuarcy_data = {refer_accuracy}") + assert abs(current_accuracy - refer_accuracy) / refer_accuracy <= 0.05 + + +if __name__ == '__main__': + tuning_log = os.path.join(args.logs_dir, f"{args.framework}-{args.model}-tune.log") + refer = get_refer_data() + if args.stage == "collect_log": + collect_log() + elif args.stage == "int8_benchmark": + check_status("int8", "INT8") + elif args.stage == "fp32_benchmark": + check_status("fp32", "FP32") + else: + raise ValueError(f"{args.stage} does not exist") diff --git a/.azure-pipelines/scripts/models/env_setup.sh b/.azure-pipelines/scripts/models/env_setup.sh new file mode 100644 index 00000000000..7443e3e9d25 --- /dev/null +++ b/.azure-pipelines/scripts/models/env_setup.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@"; do + case $i in + --yaml=*) + yaml=$(echo $i | sed "s/${PATTERN}//") + ;; + --framework=*) + framework=$(echo $i | sed "s/${PATTERN}//") + ;; + --fwk_ver=*) + fwk_ver=$(echo $i | sed "s/${PATTERN}//") + ;; + --torch_vision_ver=*) + torch_vision_ver=$(echo $i | sed "s/${PATTERN}//") + ;; + --model=*) + model=$(echo $i | sed "s/${PATTERN}//") + ;; + --model_src_dir=*) + model_src_dir=$(echo $i | sed "s/${PATTERN}//") + ;; + --dataset_location=*) + dataset_location=$(echo $i | sed "s/${PATTERN}//") + ;; + --batch_size=*) + batch_size=$(echo $i | sed "s/${PATTERN}//") + ;; + --strategy=*) + strategy=$(echo $i | sed "s/${PATTERN}//") + ;; + --new_benchmark=*) + new_benchmark=$(echo $i | sed "s/${PATTERN}//") + ;; + *) + echo "Parameter $i not recognized." + exit 1 + ;; + esac +done + +SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" +log_dir="/neural-compressor/.azure-pipelines/scripts/models" +WORK_SOURCE_DIR="/neural-compressor/examples/${framework}" +$BOLD_YELLOW && echo "processing ${framework}-${fwk_ver}-${model}" && $RESET + +$BOLD_YELLOW && echo "======= creat log_dir =========" && $RESET +if [ -d "${log_dir}/${model}" ]; then + $BOLD_GREEN && echo "${log_dir}/${model} already exists, don't need to mkdir." && $RESET +else + $BOLD_GREEN && echo "no log dir ${log_dir}/${model}, create." && $RESET + cd ${log_dir} + mkdir ${model} +fi + +$BOLD_YELLOW && echo "====== install requirements ======" && $RESET +/bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh + +cd ${WORK_SOURCE_DIR}/${model_src_dir} +pip install ruamel_yaml +pip install psutil +pip install protobuf==3.20.1 +if [[ "${framework}" == "tensorflow" ]]; then + pip install intel-tensorflow==${fwk_ver} +elif [[ "${framework}" == "pytorch" ]]; then + pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html + pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html +elif [[ "${framework}" == "onnxrt" ]]; then + pip install onnx==1.11.0 + pip install onnxruntime==${fwk_ver} +elif [[ "${framework}" == "mxnet" ]]; then + if [[ "${fwk_ver}" == "1.7.0" ]]; then + pip install mxnet==${fwk_ver}.post2 + elif [[ "${fwk_ver}" == "1.6.0" ]]; then + pip install mxnet-mkl==${mxnet_version} + else + pip install mxnet==${fwk_ver} + fi +fi + +if [ -f "requirements.txt" ]; then + sed -i '/neural-compressor/d' requirements.txt + if [ "${framework}" == "onnxrt" ]; then + sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt + fi + if [ "${framework}" == "tensorflow" ]; then + sed -i '/tensorflow==/d;/tensorflow$/d' requirements.txt + sed -i '/^intel-tensorflow/d' requirements.txt + fi + if [ "${framework}" == "mxnet" ]; then + sed -i '/mxnet==/d;/mxnet$/d;/mxnet-mkl==/d;/mxnet-mkl$/d' requirements.txt + fi + if [ "${framework}" == "pytorch" ]; then + sed -i '/torch==/d;/torch$/d;/torchvision==/d;/torchvision$/d' requirements.txt + fi + n=0 + until [ "$n" -ge 5 ]; do + python -m pip install -r requirements.txt && break + n=$((n + 1)) + sleep 5 + done + pip list +else + $BOLD_RED && echo "Not found requirements.txt file." && $RESET +fi + +$BOLD_YELLOW && echo "======== update yaml config ========" && $RESET +$BOLD_YELLOW && echo -e "\nPrint origin yaml..." && $RESET +cat ${yaml} +python ${SCRIPTS_PATH}/update_yaml_config.py \ + --yaml=${yaml} \ + --framework=${framework} \ + --dataset_location=${dataset_location} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --multi_instance='true' +$BOLD_YELLOW && echo -e "\nPrint updated yaml... " && $RESET +cat ${yaml} diff --git a/.azure-pipelines/scripts/models/generate_report.sh b/.azure-pipelines/scripts/models/generate_report.sh new file mode 100644 index 00000000000..e76cba525f4 --- /dev/null +++ b/.azure-pipelines/scripts/models/generate_report.sh @@ -0,0 +1,591 @@ +#!/bin/bash + +# WORKSPACE=. +# summaryLog=summary.log +# summaryLogLast=summary.log +# tuneLog=tuning_info.log +# tuneLogLast=tuning_info.log +# overview_log=summary_overview.log +# coverage_summary=coverage_summary.log +# nc_code_lines_summary=nc_code_lines_summary.csv +# engine_code_lines_summary=engine_code_lines_summary.csv + +#lines_coverage_threshold=80 +#branches_coverage_threshold=75 +# +#pass_status="Pass" +#fail_status="Fail" +#verify_status="Verify" + + +# shellcheck disable=SC2120 + +while [[ $# -gt 0 ]];do + key=${1} + case ${key} in + -w|--WORKSPACE) + WORKSPACE=${2} + shift 2 + ;; + --script_path) + script_path=${2} + shift 2 + ;; + --output_dir) + output_dir=${2} + shift 2 + ;; + --last_logt_dir) + last_logt_dir=${2} + shift 2 + ;; + *) + shift + ;; + esac +done + +echo "workspace: ${WORKSPACE}" +echo "script_path: ${script_path}" + +summaryLog="${WORKSPACE}/summary.log" +tuneLog="${WORKSPACE}/tuning_info.log" +echo "summaryLog: ${summaryLog}" +echo "tuneLog: ${tuneLog}" + +echo "last_logt_dir: ${last_logt_dir}" +summaryLogLast="${last_logt_dir}/summary.log" +tuneLogLast="${last_logt_dir}/tuning_info.log" +echo "summaryLogLast: ${summaryLogLast}" +echo "tuneLogLast: ${tuneLogLast}" +ghprbPullId=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER} +MR_source_branch=${SYSTEM_PULLREQUEST_SOURCEBRANCH} +MR_source_repo=${SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI} +MR_target_branch=${SYSTEM_PULLREQUEST_TARGETBRANCH} +repo_url=${BUILD_REPOSITORY_URI} +source_commit_id=${BUILD_SOURCEVERSION} +build_id=${BUILD_BUILDID} +echo "MR_source_branch: ${MR_source_branch}" +echo "MR_source_repo: ${MR_source_repo}" +echo "MR_target_branch: ${MR_target_branch}" +echo "repo_url: ${repo_url}" +echo "commit_id: ${source_commit_id}" +echo "ghprbPullId: ${ghprbPullId}" + + +function main { + generate_html_head + generate_html_body + generate_results + generate_html_footer +} + +function generate_inference { +# echo "Generating inference" + awk -v framework="${framework}" -v fw_version="${fw_version}" -v model="${model}" -v os="${os}" -v platform=${platform} -F ';' ' + BEGINE { + fp32_perf_bs = "nan"; + fp32_perf_value = "nan"; + fp32_perf_url = "nan"; + fp32_acc_bs = "nan"; + fp32_acc_value = "nan"; + fp32_acc_url = "nan"; + + int8_perf_bs = "nan"; + int8_perf_value = "nan"; + int8_perf_url = "nan"; + int8_acc_bs = "nan"; + int8_acc_value = "nan"; + int8_acc_url = "nan"; + }{ + if($1 == os && $2 == platform && $3 == framework && $4 == fw_version && $6 == model) { + // FP32 + if($5 == "FP32") { + // Performance + if($8 == "Performance") { + fp32_perf_bs = $9; + fp32_perf_value = $10; + fp32_perf_url = $11; + } + // Accuracy + if($8 == "Accuracy") { + fp32_acc_bs = $9; + fp32_acc_value = $10; + fp32_acc_url = $11; + } + } + + // INT8 + if($5 == "INT8") { + // Performance + if($8 == "Performance") { + int8_perf_bs = $9; + int8_perf_value = $10; + int8_perf_url = $11; + } + // Accuracy + if($8 == "Accuracy") { + int8_acc_bs = $9; + int8_acc_value = $10; + int8_acc_url = $11; + } + } + } + }END { + printf("%s;%s;%s;%s;", int8_perf_bs,int8_perf_value,int8_acc_bs,int8_acc_value); + printf("%s;%s;%s;%s;", fp32_perf_bs,fp32_perf_value,fp32_acc_bs,fp32_acc_value); + printf("%s;%s;%s;%s;", int8_perf_url,int8_acc_url,fp32_perf_url,fp32_acc_url); + } + ' "$1" +} + +function generate_html_core { + echo "--- current values ---" + echo ${current_values} + echo "--- last values ---" + echo ${last_values} + tuning_strategy=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $6}') + tuning_time=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $7}') + tuning_count=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $8}') + tuning_log=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $9}') + echo "${platform}${os}${framework}${fw_version}${model}New${tuning_strategy}" >> ${output_dir}/report.html + echo "${tuning_time}${tuning_count}" >> ${output_dir}/report.html + + tuning_strategy=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $6}') + tuning_time=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $7}') + tuning_count=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $8}') + tuning_log=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $9}') + + echo |awk -F ';' -v current_values="${current_values}" -v last_values="${last_values}" \ + -v tuning_strategy="${tuning_strategy}" -v tuning_time="${tuning_time}" \ + -v tuning_count="${tuning_count}" -v tuning_log="${tuning_log}" -F ';' ' + + function abs(x) { return x < 0 ? -x : x } + + function show_new_last(batch, link, value, metric) { + if(value ~/[1-9]/) { + if (metric == "perf") { + printf("%s %.2f\n",batch,link,value); + } else { + printf("%s %.2f%\n",batch,link,value*100); + } + } else { + if(link == "" || value == "N/A") { + printf(" \n"); + } else { + printf("%s Failure\n",batch,link); + } + } + } + + function compare_current(int8_result, fp32_result, metric) { + + if(int8_result ~/[1-9]/ && fp32_result ~/[1-9]/) { + if(metric == "acc") { + target = (int8_result - fp32_result) / fp32_result; + if(target >= -0.01) { + printf("%.2f %", target*100); + }else if(target < -0.05) { + printf("%.2f %", target*100); + job_status = "fail" + }else{ + printf("%.2f %", target*100); + } + }else if(metric == "perf") { + target = int8_result / fp32_result; + if(target >= 1.5) { + printf("%.2f", target); + }else if(target < 1) { + printf("%.2f", target); + job_status = "fail" + }else{ + printf("%.2f", target); + } + } + else { + target = int8_result / fp32_result; + if(target >= 2) { + printf("%.2f", target); + }else if(target < 1) { + printf("%.2f", target); + job_status = "fail" + }else{ + printf("%.2f", target); + } + } + }else { + printf(""); + } + } + + function compare_result(new_result, previous_result, metric) { + + if (new_result ~/[1-9]/ && previous_result ~/[1-9]/) { + if(metric == "acc") { + target = new_result - previous_result; + if(target > -0.00001 && target < 0.00001) { + status_png = "background-color:#90EE90"; + } else { + status_png = "background-color:#FFD2D2"; + job_status = "fail" + } + printf("%.2f %", status_png, target*100); + } else { + target = new_result / previous_result; + if(target >= 0.945) { + status_png = "background-color:#90EE90"; + } else { + status_png = "background-color:#FFD2D2"; + job_status = "fail" + } + printf("%.2f", status_png, target); + } + } else { + if(new_result == nan && previous_result == nan){ + printf(""); + } else{ + if(new_result == nan) { + job_status = "fail" + status_png = "background-color:#FFD2D2"; + printf("", status_png); + } else{ + printf(""); + } + } + } + } + + BEGIN { + job_status = "pass" + // issue list + jira_mobilenet = "https://jira01.devtools.intel.com/browse/PADDLEQ-384"; + jira_resnext = "https://jira01.devtools.intel.com/browse/PADDLEQ-387"; + jira_ssdmobilenet = "https://jira01.devtools.intel.com/browse/PADDLEQ-406"; + }{ + // Current values + split(current_values,current_value,";"); + + // Current + + // INT8 Performance results + int8_perf_batch=current_value[1] + int8_perf_value=current_value[2] + int8_perf_url=current_value[9] + show_new_last(int8_perf_batch, int8_perf_url, int8_perf_value, "perf"); + + // INT8 Accuracy results + int8_acc_batch=current_value[3] + int8_acc_value=current_value[4] + int8_acc_url=current_value[10] + show_new_last(int8_acc_batch, int8_acc_url, int8_acc_value, "acc"); + + // FP32 Performance results + fp32_perf_batch=current_value[5] + fp32_perf_value=current_value[6] + fp32_perf_url=current_value[11] + show_new_last(fp32_perf_batch, fp32_perf_url, fp32_perf_value, "perf"); + + // FP32 Accuracy results + fp32_acc_batch=current_value[7] + fp32_acc_value=current_value[8] + fp32_acc_url=current_value[12] + show_new_last(fp32_acc_batch, fp32_acc_url, fp32_acc_value, "acc"); + + // Compare Current + + compare_current(int8_perf_value, fp32_perf_value, "perf"); + compare_current(int8_acc_value, fp32_acc_value, "acc"); + + // Last values + split(last_values,last_value,";"); + + // Last + printf("\nLast%1$s%2$s%3$s", tuning_strategy, tuning_time, tuning_count, tuning_log); + + // Show last INT8 Performance results + last_int8_perf_batch=last_value[1] + last_int8_perf_value=last_value[2] + last_int8_perf_url=last_value[9] + show_new_last(last_int8_perf_batch, last_int8_perf_url, last_int8_perf_value, "perf"); + + // Show last INT8 Accuracy results + last_int8_acc_batch=last_value[3] + last_int8_acc_value=last_value[4] + last_int8_acc_url=last_value[10] + show_new_last(last_int8_acc_batch, last_int8_acc_url, last_int8_acc_value, "acc"); + + // Show last FP32 Performance results + last_fp32_perf_batch=last_value[5] + last_fp32_perf_value=last_value[6] + last_fp32_perf_url=last_value[11] + show_new_last(last_fp32_perf_batch, last_fp32_perf_url, last_fp32_perf_value, "perf"); + + // Show last FP32 Accuracy results + last_fp32_acc_batch=last_value[7] + last_fp32_acc_value=last_value[8] + last_fp32_acc_url=last_value[12] + show_new_last(last_fp32_acc_batch, last_fp32_acc_url, last_fp32_acc_value, "acc"); + + printf("") + + // current vs last + printf("\nNew/Last"); + + // Compare INT8 Performance results + compare_result(int8_perf_value, last_int8_perf_value,"perf"); + + // Compare INT8 Accuracy results + compare_result(int8_acc_value, last_int8_acc_value, "acc"); + + // Compare FP32 Performance results + compare_result(fp32_perf_value, last_fp32_perf_value, "perf"); + + // Compare INT8 Performance results + compare_result(fp32_acc_value, last_fp32_acc_value, "acc"); + + printf("\n"); + + } END{ + printf("\n%s", job_status); + } + ' >> ${output_dir}/report.html + job_state=$(tail -1 ${WORKSPACE}/report.html) + sed -i '$s/.*//' ${WORKSPACE}/report.html + + if [ ${job_state} == 'fail' ]; then + echo "====== perf_reg ======" + echo "##vso[task.setvariable variable=is_perf_reg]true" + fi +} + +function generate_results { + echo "Generating tuning results" + oses=$(sed '1d' ${summaryLog} |cut -d';' -f1 | awk '!a[$0]++') + echo ${oses} + + for os in ${oses[@]} + do + platforms=$(sed '1d' ${summaryLog} |grep "^${os}" |cut -d';' -f2 | awk '!a[$0]++') + echo ${platforms} + for platform in ${platforms[@]} + do + frameworks=$(sed '1d' ${summaryLog} |grep "^${os};${platform}" |cut -d';' -f3 | awk '!a[$0]++') + echo ${frameworks} + for framework in ${frameworks[@]} + do + fw_versions=$(sed '1d' ${summaryLog} |grep "^${os};${platform};${framework}" |cut -d';' -f4 | awk '!a[$0]++') + echo ${fw_versions} + for fw_version in ${fw_versions[@]} + do + models=$(sed '1d' ${summaryLog} |grep "^${os};${platform};${framework};${fw_version}" |cut -d';' -f6 | awk '!a[$0]++') + echo ${models} + for model in ${models[@]} + do + echo "--- processing model ---" + echo ${model} + current_values=$(generate_inference ${summaryLog}) + echo "| current value |" + echo ${current_values} + last_values=$(generate_inference ${summaryLogLast}) + echo "| last value |" + echo ${last_values} + + generate_html_core ${current_values} ${last_values} + done + done + done + done + done +} + +function generate_html_body { +MR_TITLE='' +Test_Info_Title='' +Test_Info='' + +if [ "${qtools_branch}" == "" ]; +then + commit_id=$(echo ${ghprbActualCommit} |awk '{print substr($1,1,7)}') + + MR_TITLE="[ PR-${ghprbPullId} ]" + Test_Info_Title="Source Branch Target Branch Commit " + Test_Info="${MR_source_branch} ${MR_target_branch} ${source_commit_id:0:6}" +else + Test_Info_Title="Test Branch Commit ID " + Test_Info="${qtools_branch} ${qtools_commit} " +fi + +cat >> ${output_dir}/report.html << eof + + +
+

Neural Compressor Tuning Tests ${MR_TITLE} + [ Job-${build_id} + ]

+

Test Status: ${Jenkins_job_status}

+

Summary

+ + + + ${Test_Info_Title} + + + + ${Test_Info} + +
Repo
neural-compressor
+eof + + +echo "Generating benchmarks table" +cat >> ${output_dir}/report.html << eof +

Benchmark

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +eof +} + +function generate_html_footer { + + cat >> ${output_dir}/report.html << eof + + + + +
PlatformSystemFrameworkVersionModelVSTuning
Strategy		Tuning
Time(s)		Tuning
Count		INT8FP32Ratio
bsimgs/sbstop1bsimgs/sbstop1Throughput
INT8/FP32>=2		Accuracy
(INT8-FP32)/FP32>=-0.01
Note: All data tested on TensorFlow Dedicated Server.
+
+ + +eof +} + +function generate_html_head { + +cat > ${output_dir}/report.html << eof + + + + + + Daily Tests - TensorFlow - Jenkins + + + +eof + +} + +main diff --git a/.azure-pipelines/scripts/models/run_benchmark_common.sh b/.azure-pipelines/scripts/models/run_benchmark_common.sh new file mode 100644 index 00000000000..ae5b8a1af36 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_benchmark_common.sh @@ -0,0 +1,129 @@ +#!/bin/bash +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' +SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" + +for i in "$@"; do + case $i in + --framework=*) + framework=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --input_model=*) + input_model=`echo $i | sed "s/${PATTERN}//"`;; + --benchmark_cmd=*) + benchmark_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --log_dir=*) + log_dir=`echo $i | sed "s/${PATTERN}//"`;; + --new_benchmark=*) + new_benchmark=`echo $i | sed "s/${PATTERN}//"`;; + --precision=*) + precision=`echo $i | sed "s/${PATTERN}//"`;; + --stage=*) + stage=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +$BOLD_YELLOW && echo "-------- run_benchmark_common --------" && $RESET + +main() { + # run accuracy + # USE_TUNE_ACC==true means using accuracy results from tuning log + if [ "${USE_TUNE_ACC}" == "false" ]; then + run_accuracy + fi + + # run performance + if [ "${PERF_STABLE_CHECK}" == "false" ]; then + run_performance + else + max_loop=3 + for ((iter = 0; iter < ${max_loop}; iter++)); do + run_performance + { + check_perf_gap + exit_code=$? + } || true + + if [ ${exit_code} -ne 0 ]; then + $BOLD_RED && echo "FAILED with performance gap!!" && $RESET + else + $BOLD_GREEN && echo "SUCCEED!!" && $RESET + break + fi + done + exit ${exit_code} + fi +} + +function check_perf_gap() { + python -u ${SCRIPTS_PATH}/collect_log_model.py \ + --framework=${framework} \ + --fwk_ver=${fwk_ver} \ + --model=${model} \ + --logs_dir="${log_dir}" \ + --output_dir="${log_dir}" \ + --build_id=${BUILD_BUILDID} \ + --stage=${stage} +} + +function run_performance() { + cmd="${benchmark_cmd} --input_model=${input_model}" + if [ "${new_benchmark}" == "true" ]; then + $BOLD_YELLOW && echo "run with internal benchmark..." && $RESET + eval ${cmd} 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log + else + $BOLD_YELLOW && echo "run with external multiInstance benchmark..." && $RESET + multiInstance + fi +} + +function run_accuracy() { + $BOLD_YELLOW && echo "run tuning accuracy in precision ${precision}" && $RESET + eval "${benchmark_cmd} --input_model=${input_model} --mode=accuracy" 2>&1 | tee ${log_dir}/${framework}-${model}-accuracy-${precision}.log +} + +function multiInstance() { + ncores_per_socket=${ncores_per_socket:=$(lscpu | grep 'Core(s) per socket' | cut -d: -f2 | xargs echo -n)} + $BOLD_YELLOW && echo "Executing multi instance benchmark" && $RESET + ncores_per_instance=4 + $BOLD_YELLOW && echo "ncores_per_socket=${ncores_per_socket}, ncores_per_instance=${ncores_per_instance}" && $RESET + + logFile="${log_dir}/${framework}-${model}-performance-${precision}" + benchmark_pids=() + + for ((j = 0; $j < ${ncores_per_socket}; j = $(($j + ${ncores_per_instance})))); do + end_core_num=$((j + ncores_per_instance - 1)) + if [ ${end_core_num} -ge ${ncores_per_socket} ]; then + end_core_num=$((ncores_per_socket - 1)) + fi + numactl -m 0 -C "${j}-${end_core_num}" ${cmd} 2>&1 | tee ${logFile}-${ncores_per_socket}-${ncores_per_instance}-${j}.log & + benchmark_pids+=($!) + done + + status="SUCCESS" + for pid in "${benchmark_pids[@]}"; do + wait $pid + exit_code=$? + $BOLD_YELLOW && echo "Detected exit code: ${exit_code}" && $RESET + if [ ${exit_code} == 0 ]; then + $BOLD_GREEN && echo "Process ${pid} succeeded" && $RESET + else + $BOLD_RED && echo "Process ${pid} failed" && $RESET + status="FAILURE" + fi + done + + $BOLD_YELLOW && echo "Benchmark process status: ${status}" && $RESET + if [ ${status} == "FAILURE" ]; then + $BOLD_RED && echo "Benchmark process returned non-zero exit code." && $RESET + exit 1 + fi +} + +main diff --git a/.azure-pipelines/scripts/models/run_model_trigger_common.sh b/.azure-pipelines/scripts/models/run_model_trigger_common.sh new file mode 100644 index 00000000000..d0c89560416 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_model_trigger_common.sh @@ -0,0 +1,129 @@ +#!/bin/bash +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --yaml=*) + yaml=`echo $i | sed "s/${PATTERN}//"`;; + --framework=*) + framework=`echo $i | sed "s/${PATTERN}//"`;; + --fwk_ver=*) + fwk_ver=`echo $i | sed "s/${PATTERN}//"`;; + --torch_vision_ver=*) + torch_vision_ver=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --model_src_dir=*) + model_src_dir=`echo $i | sed "s/${PATTERN}//"`;; + --dataset_location=*) + dataset_location=`echo $i | sed "s/${PATTERN}//"`;; + --input_model=*) + input_model=`echo $i | sed "s/${PATTERN}//"`;; + --batch_size=*) + batch_size=`echo $i | sed "s/${PATTERN}//"`;; + --strategy=*) + strategy=`echo $i | sed "s/${PATTERN}//"`;; + --new_benchmark=*) + new_benchmark=`echo $i | sed "s/${PATTERN}//"`;; + --tuning_cmd=*) + tuning_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --benchmark_cmd=*) + benchmark_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +log_dir="/neural-compressor/.azure-pipelines/scripts/models" +WORK_SOURCE_DIR="/neural-compressor/examples/${framework}" +SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" +$BOLD_YELLOW && echo "processing ${framework}-${fwk_ver}-${model}" && $RESET +$BOLD_YELLOW && echo "benchmark_cmd is ${benchmark_cmd}" && $RESET + +if [ "${mode}" == "env_setup" ]; then + /bin/bash env_setup.sh \ + --yaml=${yaml} \ + --framework=${framework} \ + --fwk_ver=${fwk_ver} \ + --torch_vision_ver=${torch_vision_ver} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} +elif [ "${mode}" == "tuning" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "tuning_cmd is ${tuning_cmd}" && $RESET + $BOLD_YELLOW && echo "======== run tuning ========" && $RESET + /bin/bash ${SCRIPTS_PATH}/run_tuning_common.sh \ + --framework=${framework} \ + --model=${model} \ + --tuning_cmd="${tuning_cmd}" \ + --log_dir="${log_dir}/${model}" \ + --input_model=${input_model} \ + --strategy=${strategy} \ + 2>&1 | tee -a ${log_dir}/${model}/${framework}-${model}-tune.log + $BOLD_YELLOW && echo "====== check tuning status. ======" && $RESET + control_phrase="model which meet accuracy goal." + if [ $(grep "${control_phrase}" ${log_dir}/${model}/${framework}-${model}-tune.log | wc -l) == 0 ];then + $BOLD_RED && echo "====== tuning FAILED!! ======" && $RESET; exit 1 + fi + if [ $(grep "${control_phrase}" ${log_dir}/${model}/${framework}-${model}-tune.log | grep "Not found" | wc -l) == 1 ];then + $BOLD_RED && echo "====== tuning FAILED!! ======" && $RESET; exit 1 + fi + $BOLD_GREEN && echo "====== tuning SUCCEED!! ======" && $RESET +elif [ "${mode}" == "fp32_benchmark" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "benchmark_cmd is ${benchmark_cmd}" && $RESET + $BOLD_YELLOW && echo "====== run benchmark fp32 =======" && $RESET + /bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh \ + --framework=${framework} \ + --model=${model} \ + --input_model=${input_model} \ + --benchmark_cmd="${benchmark_cmd}" \ + --log_dir="${log_dir}/${model}" \ + --new_benchmark=${new_benchmark} \ + --precision="fp32" \ + --stage=${mode} +elif [ "${mode}" == "int8_benchmark" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "benchmark_cmd is ${benchmark_cmd}" && $RESET + $BOLD_YELLOW && echo "====== run benchmark int8 =======" && $RESET + if [[ "${framework}" == "onnxrt" ]]; then + model_name="${log_dir}/${model}/${framework}-${model}-tune.onnx" + elif [[ "${framework}" == "mxnet" ]]; then + model_name="${log_dir}/${model}" + elif [[ "${framework}" == "tensorflow" ]]; then + model_name="${log_dir}/${model}/${framework}-${model}-tune.pb" + elif [[ "${framework}" == "pytorch" ]]; then + model_name=${input_model} + benchmark_cmd="${benchmark_cmd} --int8=true" + fi + /bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh \ + --framework=${framework} \ + --model=${model} \ + --input_model="${model_name}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --log_dir="${log_dir}/${model}" \ + --new_benchmark=${new_benchmark} \ + --precision="int8" \ + --stage=${mode} +elif [ "${mode}" == "collect_log" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "====== collect logs of model ${model} =======" && $RESET + python -u ${SCRIPTS_PATH}/collect_log_model.py \ + --framework=${framework} \ + --fwk_ver=${fwk_ver} \ + --model=${model} \ + --logs_dir="${log_dir}/${model}" \ + --output_dir="${log_dir}/${model}" \ + --build_id=${BUILD_BUILDID} + $BOLD_YELLOW && echo "====== Finish collect logs =======" && $RESET +fi diff --git a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh new file mode 100644 index 00000000000..8bf3b293fc2 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -eo pipefail +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="mxnet" +FRAMEWORK_VERSION="1.7.0" + + +# ======== set up config for mxnet models ======== +if [ "${model}" == "resnet50v1" ]; then + model_src_dir="image_recognition/cnn_models/quantization/ptq" + dataset_location="/tf_dataset/mxnet/val_256_q90.rec" + input_model="/tf_dataset/mxnet/resnet50_v1" + yaml="cnn.yaml" + strategy="mse" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=resnet50_v1 --dataset_location=${dataset_location} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=resnet50_v1 --dataset_location=${dataset_location} --batch_size=1 --iters=500 --mode=benchmark" +fi + + +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh new file mode 100644 index 00000000000..a69852f01a5 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -eo pipefail +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="onnxrt" +FRAMEWORK_VERSION="1.12.1" + + +# ======== set up config for onnxrt models ======== +if [ "${model}" == "resnet50-v1-12" ]; then + model_src_dir="image_recognition/onnx_model_zoo/resnet50/quantization/ptq" + dataset_location="/tf_dataset2/datasets/imagenet/ImagenetRaw/ImagenetRaw_small_5000/ILSVRC2012_img_val" + input_model="/tf_dataset2/models/onnx/resnet50-v1-12/resnet50-v1-12.onnx" + yaml="resnet50_v1_5.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "bert_base_MRPC_static" ]; then + model_src_dir="language_translation/bert/quantization/ptq" + dataset_location="/tf_dataset/pytorch/glue_data/MRPC" + input_model="/tf_dataset2/models/onnx/bert_base_MRPC/bert.onnx" + yaml="bert_static.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "bert_base_MRPC_dynamic" ]; then + model_src_dir="language_translation/bert/quantization/ptq" + dataset_location="/tf_dataset/pytorch/glue_data/MRPC" + input_model="/tf_dataset2/models/onnx/bert_base_MRPC/bert.onnx" + yaml="bert_dynamic.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "distilbert_base_MRPC_qdq" ]; then + model_src_dir="language_translation/distilbert/quantization/ptq" + dataset_location="/tf_dataset/pytorch/glue_data/MRPC" + input_model="/tf_dataset2/models/onnx/distilbert_base_MRPC/distilbert-base-uncased.onnx" + yaml="distilbert_qdq.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +fi + + +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh new file mode 100644 index 00000000000..5cd776816f4 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -eo pipefail +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="pytorch" +FRAMEWORK_VERSION="1.12.0+cpu" +TORCH_VISION_VERSION="0.13.0+cpu" + + +# ======== set up config for pytorch models ======== +if [ "${model}" == "resnet18" ]; then + model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/eager" + dataset_location="/tf_dataset2/datasets/mini-imageraw" + input_model="" + yaml="conf.yaml" + strategy="bayesian" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=500" +elif [ "${model}" == "resnet18_fx" ]; then + model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/fx/" + dataset_location="/tf_dataset2/datasets/mini-imageraw" + input_model="" + yaml="conf.yaml" + strategy="basic" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=500" +fi + + +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --torch_vision_ver=${TORCH_VISION_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh new file mode 100644 index 00000000000..b3eee910900 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh @@ -0,0 +1,118 @@ +#!/bin/bash +set -eo pipefail +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="tensorflow" +FRAMEWORK_VERSION="2.9.1" + +# ======== set up config for tensorflow models ======== +if [ "${model}" == "resnet50v1.5" ]; then + model_src_dir="image_recognition/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/dataset/TF_mini_imagenet" + input_model="/tf_dataset/pre-trained-models/resnet50v1_5/fp32/resnet50_v1.pb" + yaml="resnet50_v1_5.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "ssd_resnet50_v1" ];then + model_src_dir="object_detection/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/tensorflow/mini-coco-100.record" + input_model="/tf_dataset/pre-train-model-oob/object_detection/ssd_resnet50_v1/frozen_inference_graph.pb" + yaml="ssd_resnet50_v1.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "ssd_mobilenet_v1_ckpt" ];then + model_src_dir="object_detection/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/tensorflow/mini-coco-100.record" + input_model="/tf_dataset/pre-train-model-oob/object_detection/ssd_mobilenet_v1" + yaml="ssd_mobilenet_v1.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "inception_v1" ]; then + model_src_dir="image_recognition/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/dataset/TF_mini_imagenet" + input_model="/tf_dataset/pre-train-model-slim/pbfile/frozen_pb/frozen_inception_v1.pb" + yaml="inception_v1.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "darknet19" ]; then + model_src_dir="oob_models/quantization/ptq" + dataset_location="" + input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/PublicInHouse/classification/darknet19/darknet19.pb" + yaml="config.yaml" + strategy="basic" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500" +elif [ "${model}" == "densenet-121" ]; then + model_src_dir="oob_models/quantization/ptq" + dataset_location="" + input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/classification/densenet/121/tf/densenet-121.pb" + yaml="config.yaml" + strategy="basic" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500" +elif [ "${model}" == "resnet-101" ]; then + model_src_dir="oob_models/quantization/ptq" + dataset_location="" + input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/classification/resnet/v1/101/tf/resnet-101.pb" + yaml="config.yaml" + strategy="basic" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500" +elif [ "${model}" == "resnet50_fashion" ]; then + model_src_dir="image_recognition/keras_models/resnet50_fashion/quantization/ptq" + dataset_location="/tf_dataset2/datasets/mnist/FashionMNIST_small" + input_model="/tf_dataset2/models/tensorflow/resnet50_fashion" + yaml="resnet50_fashion.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +fi + + +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_tuning_common.sh b/.azure-pipelines/scripts/models/run_tuning_common.sh new file mode 100644 index 00000000000..fbb68d65605 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_tuning_common.sh @@ -0,0 +1,50 @@ +#!/bin/bash +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +starttime=`date +'%Y-%m-%d %H:%M:%S'` + +for i in "$@" +do + case $i in + --framework=*) + framework=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --input_model=*) + input_model=`echo $i | sed "s/${PATTERN}//"`;; + --tuning_cmd=*) + tuning_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --log_dir=*) + log_dir=`echo $i | sed "s/${PATTERN}//"`;; + --strategy=*) + strategy=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +# run tuning +if [ "${framework}" == "onnxrt" ]; then + output_model=${log_dir}/${framework}-${model}-tune.onnx +elif [ "${framework}" == "mxnet" ]; then + output_model=${log_dir}/resnet50_v1 +else + output_model=${log_dir}/${framework}-${model}-tune.pb +fi + +$BOLD_YELLOW && echo -e "-------- run_tuning_common --------" && $RESET +$BOLD_YELLOW && echo ${tuning_cmd} && $RESET + +eval "/usr/bin/time -v ${tuning_cmd} --output_model=${output_model}" + +$BOLD_YELLOW && echo "====== finish tuning. echo information. ======" && $RESET +endtime=`date +'%Y-%m-%d %H:%M:%S'` +start_seconds=$(date --date="$starttime" +%s); +end_seconds=$(date --date="$endtime" +%s); +$BOLD_GREEN && echo "Tuning time spend: "$((end_seconds-start_seconds))"s " && $RESET +$BOLD_GREEN && echo "Tuning strategy: ${strategy}" && $RESET +$BOLD_GREEN && echo "Total resident size (kbytes): $(cat /proc/meminfo | grep 'MemTotal' | sed 's/[^0-9]//g')" && $RESET diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py new file mode 100644 index 00000000000..c305134e18d --- /dev/null +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -0,0 +1,309 @@ +import argparse +import re +import os +import psutil +from typing import Optional, Union + +import platform + +system = platform.system() +try: + import ruamel.yaml as yaml +except: + import ruamel_yaml as yaml + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--yaml", type=str, required=True, help="Path to yaml config.") + parser.add_argument("--framework", type=str, required=True, help="Framework of model.") + parser.add_argument("--dataset_location", type=str, required=True, help="Location of dataset used for model.") + parser.add_argument("--strategy", type=str, required=False, help="Strategy to update.") + parser.add_argument("--batch_size", type=int, required=False, help="Batch size.") + parser.add_argument("--new_benchmark", type=str, required=False, help="Whether to modify benchmark config.") + parser.add_argument("--multi_instance", type=str, required=False, help="Whether to eval in multi-instance.") + return parser.parse_args() + + +def update_yaml_dataset(yaml, framework, dataset_location): + if not os.path.isfile(yaml): + raise Exception(f"Not found yaml config at '{yaml}' location.") + + print("Reading config") + with open(yaml, "r") as config: + lines = config.readlines() + + # Update dataset + if framework != "pytorch": + val_txt_location = os.path.dirname(dataset_location) + f"{os.path.sep}" + "val.txt" + + patterns = { + "root_path": { + "pattern": r'root:.*/path/to/(calibration|evaluation)/dataset/?', + "replacement": f"root: {dataset_location}", + }, + "data_path": { + "pattern": r'data_path:.*/path/to/(calibration|evaluation)/dataset/?', + "replacement": f"data_path: {dataset_location}", + }, + "image_list": { + "pattern": r'image_list:.*/path/to/(calibration|evaluation)/label/?', + "replacement": f"image_list: {val_txt_location}", + }, + "data_dir": { + "pattern": r'data_dir:.*/path/to/dataset/?', + "replacement": f"data_dir: {dataset_location}", + }, + } + print("======= update_yaml_dataset =======") + with open(yaml, "w") as config: + for line in lines: + for key, key_patterns in patterns.items(): + if re.search(key_patterns["pattern"], line): + print(f"Replacing {key} key.") + line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) + config.write(line) + + else: + val_dataset = dataset_location + f"{os.path.sep}" + "val" + train_dataset = dataset_location + f"{os.path.sep}" + "train" + patterns = { + "calibration_dataset": { + "pattern": r'root:.*/path/to/calibration/dataset/?', + "replacement": f"root: {train_dataset}", + }, + "evaluation_dataset": { + "pattern": r'root:.*/path/to/evaluation/dataset/?', + "replacement": f"root: {val_dataset}", + }, + } + + print("======= update_yaml_dataset =======") + with open(yaml, "w") as config: + for line in lines: + for key, key_patterns in patterns.items(): + if re.search(key_patterns["pattern"], line): + print(f"Replacing {key} key.") + line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) + config.write(line) + + +def update_yaml_config_tuning(yaml_file, strategy = None, mode = None, batch_size = None, iteration = None, + max_trials = None, algorithm = None, timeout = None, strategy_token = None, + sampling_size = None, dtype = None, tf_new_api = None): + with open(yaml_file) as f: + yaml_config = yaml.round_trip_load(f, preserve_quotes=True) + + if algorithm: + try: + model_wise = yaml_config.get("quantization", {}).get("model_wise", {}) + prev_activation = model_wise.get("activation", {}) + if not prev_activation: + model_wise.update({"activation": {}}) + prev_activation = model_wise.get("activation", {}) + prev_activation.update({"algorithm": algorithm}) + except Exception as e: + print(f"[ WARNING ] {e}") + + if timeout: + try: + exit_policy = yaml_config.get("tuning", {}).get("exit_policy", {}) + prev_timeout = exit_policy.get("timeout", None) + exit_policy.update({"timeout": timeout}) + print(f"Changed {prev_timeout} to {timeout}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if strategy and strategy != "basic": # Workaround for PyTorch huggingface models (`sed` in run_tuning.sh) + try: + tuning_config = yaml_config.get("tuning", {}) + prev_strategy = tuning_config.get("strategy", {}) + if not prev_strategy: + tuning_config.update({"strategy": {}}) + prev_strategy = tuning_config.get("strategy", {}) + strategy_name = prev_strategy.get("name", None) + prev_strategy.update({"name": strategy}) + if strategy == "sigopt": + prev_strategy.update({ + "sigopt_api_token": strategy_token, + "sigopt_project_id": "lpot", + "sigopt_experiment_name": "lpot-tune", + }) + if strategy == "hawq": + prev_strategy.update({"loss": "CrossEntropyLoss"}) + print(f"Changed {strategy_name} to {strategy}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if max_trials and max_trials > 0: + try: + tuning_config = yaml_config.get("tuning", {}) + prev_exit_policy = tuning_config.get("exit_policy", {}) + if not prev_exit_policy: + tuning_config.update({"exit_policy": { + "max_trials": max_trials + }}) + else: + prev_max_trials = prev_exit_policy.get("max_trials", None) + prev_exit_policy.update({"max_trials": max_trials}) + print(f"Changed {prev_max_trials} to {max_trials}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if mode == 'accuracy': + try: + # delete performance part in yaml if exist + performance = yaml_config.get("evaluation", {}).get("performance", {}) + if performance: + yaml_config.get("evaluation", {}).pop("performance", {}) + # accuracy batch_size replace + if batch_size: + try: + dataloader = yaml_config.get("evaluation", {}).get("accuracy", {}).get("dataloader", {}) + prev_batch_size = dataloader.get("batch_size", None) + dataloader.update({"batch_size": batch_size}) + print(f"Changed accuracy batch size from {prev_batch_size} to {batch_size}") + except Exception as e: + print(f"[ WARNING ] {e}") + except Exception as e: + print(f"[ WARNING ] {e}") + elif mode: + try: + # delete accuracy part in yaml if exist + accuracy = yaml_config.get("evaluation", {}).get("accuracy", {}) + if accuracy: + yaml_config.get("evaluation", {}).pop("accuracy", {}) + # performance iteration replace + if iteration: + try: + performance = yaml_config.get("evaluation", {}).get("performance", {}) + prev_iteration = performance.get("iteration", None) + performance.update({"iteration": iteration}) + print(f"Changed performance batch size from {prev_iteration} to {iteration}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if batch_size and mode == 'latency': + try: + dataloader = yaml_config.get("evaluation", {}).get("performance", {}).get("dataloader", {}) + prev_batch_size = dataloader.get("batch_size", None) + dataloader.update({"batch_size": batch_size}) + print(f"Changed accuracy batch size from {prev_batch_size} to {batch_size}") + except Exception as e: + print(f"[ WARNING ] {e}") + + except Exception as e: + print(f"[ WARNING ] {e}") + + if sampling_size: + try: + calibration = yaml_config.get("quantization", {}).get("calibration", {}) + prev_sampling_size = calibration.get("sampling_size", None) + calibration.update({"sampling_size": sampling_size}) + print(f"Changed calibration sampling size from {prev_sampling_size} to {sampling_size}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if dtype: + try: + quantization = yaml_config.get("quantization", {}) + prev_dtype = quantization.get("dtype", None) + quantization.update({"dtype": dtype}) + print(f"Changed dtype from {prev_dtype} to {dtype}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if tf_new_api == "true": + try: + model = yaml_config.get("model", {}) + prev_framework = model.get("framework", None) + model.update({"framework": "inteltensorflow"}) + print(f"Changed framework from {prev_framework} to inteltensorflow") + except Exception as e: + print(f"[ WARNING ] {e}") + + print(f"====== update_yaml_config_tuning ========") + + yaml_content = yaml.round_trip_dump(yaml_config) + + with open(yaml_file, 'w') as output_file: + output_file.write(yaml_content) + + +def update_yaml_config_benchmark_acc(yaml_path: str, batch_size = None): + with open(yaml_path) as f: + yaml_config = yaml.round_trip_load(f, preserve_quotes=True) + try: + accuracy = yaml_config.get("evaluation", {}).get("accuracy", {}) + if not accuracy: + raise AttributeError + dataloader = accuracy.get('dataloader', {}) + if dataloader: + dataloader.update({'batch_size': batch_size}) + configs = accuracy.get('configs', {}) + if configs: + del accuracy['configs'] + except Exception as e: + print(f"[ WARNING ] {e}") + + print(f"====== update_yaml_config_benchmark_acc ========") + + yaml_content = yaml.round_trip_dump(yaml_config) + + with open(yaml_path, 'w') as output_file: + output_file.write(yaml_content) + + +def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_instance = None): + # Get cpu information for multi-instance + total_cores = psutil.cpu_count(logical=False) + total_sockets = 1 + ncores_per_socket = total_cores / total_sockets + ncores_per_instance = ncores_per_socket + iters = 100 + + if multi_instance=='true': + ncores_per_instance = 4 + iters = 500 + + with open(yaml_path) as f: + yaml_config = yaml.round_trip_load(f, preserve_quotes=True) + try: + performance = yaml_config.get("evaluation", {}).get("performance", {}) + if not performance: + raise AttributeError + dataloader = performance.get('dataloader', {}) + if dataloader: + dataloader.update({'batch_size': batch_size}) + performance.update({'iteration': iters}) + configs = performance.get('configs', {}) + if not configs: + raise AttributeError + else: + configs.update({ + 'cores_per_instance': int(ncores_per_instance), + 'num_of_instance': int(ncores_per_socket // ncores_per_instance) + }) + for attr in ['intra_num_of_threads', 'inter_num_of_threads', 'kmp_blocktime']: + if configs.get(attr): + del configs[attr] + print(configs) + except Exception as e: + print(f"[ WARNING ] {e}") + + print(f"====== update_yaml_config_benchmark_perf ========") + + yaml_content = yaml.round_trip_dump(yaml_config) + + with open(yaml_path, 'w') as output_file: + output_file.write(yaml_content) + + +if __name__ == "__main__": + args = parse_args() + update_yaml_dataset(args.yaml, args.framework, args.dataset_location) + update_yaml_config_tuning(args.yaml, strategy=args.strategy) + print('===== multi_instance={} ===='.format(args.multi_instance)) + if args.new_benchmark=='true': + update_yaml_config_benchmark_acc(args.yaml, batch_size=args.batch_size) + update_yaml_config_benchmark_perf(args.yaml, batch_size=args.batch_size, multi_instance=args.multi_instance) diff --git a/.azure-pipelines/scripts/ut/collect_log.sh b/.azure-pipelines/scripts/ut/collect_log.sh new file mode 100644 index 00000000000..58900c280d4 --- /dev/null +++ b/.azure-pipelines/scripts/ut/collect_log.sh @@ -0,0 +1,61 @@ +pip install coverage +export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.file +coverage_log="/neural-compressor/log_dir/coverage_log" +coverage_log_base="/neural-compressor/log_dir/coverage_log_base" +coverage_compare="/neural-compressor/log_dir/coverate_compare.html" +cd /neural-compressor/log_dir +echo "collect coverage for PR branch" +mkdir -p coverage_PR +cp ut-coverage-adaptor/.coverage.adaptor ./coverage_PR/ +cp ut-coverage-tfnewapi/.coverage.tfnewapi ./coverage_PR/ +cp ut-coverage-others/.coverage.others ./coverage_PR/ +cp ut-coverage-ipex/.coverage.ipex ./coverage_PR/ +cd coverage_PR +coverage combine --keep --rcfile=${COVERAGE_RCFILE} +cp .coverage /neural-compressor/.coverage +cd /neural-compressor +coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log} +coverage html -d log_dir/coverage_PR/htmlcov --rcfile=${COVERAGE_RCFILE} +coverage xml -o log_dir/coverage_PR/coverage.xml --rcfile=${COVERAGE_RCFILE} +ls -l log_dir/coverage_PR/htmlcov +echo "collect coverage for baseline" +coverage erase +cd /neural-compressor/log_dir +mkdir -p coverage_base +cp ut-coverage-adaptor-base/.coverage.adaptor ./coverage_base/ +cp ut-coverage-tfnewapi-base/.coverage.tfnewapi ./coverage_base/ +cp ut-coverage-others-base/.coverage.others ./coverage_base/ +cp ut-coverage-ipex-base/.coverage.ipex ./coverage_base/ +cd coverage_base +coverage combine --keep --rcfile=${COVERAGE_RCFILE} +cp .coverage /neural-compressor/.coverage +cd /neural-compressor +coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log_base} +coverage html -d log_dir/coverage_base/htmlcov --rcfile=${COVERAGE_RCFILE} +coverage xml -o log_dir/coverage_base/coverage.xml --rcfile=${COVERAGE_RCFILE} +ls -l log_dir/coverage_base/htmlcov +echo "compare coverage" +coverage_PR_total=$(cat ${coverage_log} | grep TOTAL | awk '{print $NF}' | sed "s|%||g") +coverage_base_total=$(cat ${coverage_log_base} | grep TOTAL | awk '{print $NF}' | sed "s|%||g") +echo "clear upload path" +rm -fr log_dir/coverage_PR/.coverage* +rm -fr log_dir/coverage_base/.coverage* +rm -fr log_dir/ut-coverage-* +if [[ ${coverage_PR_total} -lt ${coverage_base_total} ]]; then + decreate=$(($coverage_PR_total - $coverage_base_total)) + rate=$(awk 'BEGIN{printf "%.2f%\n",'$decreate/100'}') + echo "Unit Test failed with covereage decrese ${rate}%" + echo "compare coverage to give detail info" + bash -x /neural-compressor/.azure-pipelines/scripts/ut/compare_coverage.sh ${coverage_compare} ${coverage_log} ${coverage_log_base} "FAILED" + exit 1 +else + echo "Unit Test success with coverage ${coverage_PR_total}%" + echo "compare coverage to give detail info" + bash -x /neural-compressor/.azure-pipelines/scripts/ut/compare_coverage.sh ${coverage_compare} ${coverage_log} ${coverage_log_base} "SUCCESS" + #sed "1i\Unit Test success with coverage ${coverage_PR_total}\n" ${coverage_log} +fi + + +#rm -r ${coverage_log} +#rm -r ${coverage_log_base} + diff --git a/.azure-pipelines/scripts/ut/compare_coverage.sh b/.azure-pipelines/scripts/ut/compare_coverage.sh new file mode 100644 index 00000000000..9ddb0e932b0 --- /dev/null +++ b/.azure-pipelines/scripts/ut/compare_coverage.sh @@ -0,0 +1,237 @@ +output_file=$1 +coverage_pr_log=$2 +coverage_base_log=$3 +coverage_status=$4 +module_name="neural_compressor" +[[ ! -f $coverage_pr_log ]] && exit 1 +[[ ! -f $coverage_base_log ]] && exit 1 +file_name="./coverage_compare" +sed -i "s|\/usr.*${module_name}\/||g" $coverage_pr_log +sed -i "s|\/usr.*${module_name}\/||g" $coverage_base_log +diff $coverage_pr_log $coverage_base_log > diff_file +[[ $? == 0 ]] && exit 0 +grep -Po "[<,>,\d].*" diff_file | awk '{print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7}' | sed "/Name/d" | sed "/TOTAL/d" |sed "/---/d" > $file_name +[[ ! -s $file_name ]] && exit 0 +[[ -f $output_file ]] && rm -f $output_file +touch $output_file + +function generate_html_head { + +cat > ${output_file} << eof + + + + + + Daily Tests - TensorFlow - Jenkins + + + +eof +} + +function main { + generate_html_head + # generate table head + PR_stmt=$(grep "TOTAL" $coverage_pr_log | awk '{print $2}') + PR_miss=$(grep "TOTAL" $coverage_pr_log | awk '{print $3}') + PR_cover=$(grep "TOTAL" $coverage_pr_log | awk '{print $NF}') + BASE_stmt=$(grep "TOTAL" $coverage_base_log | awk '{print $2}') + BASE_miss=$(grep "TOTAL" $coverage_base_log | awk '{print $3}') + BASE_cover=$(grep "TOTAL" $coverage_base_log | awk '{print $NF}') + echo """ + +
+

Coverage Summary : ${coverage_status}

+ + + + + + + + + + + + + + + + + + + +
CommitStatementsMissCoverage
PR ${PR_stmt} ${PR_miss} ${PR_cover}
BASE ${BASE_stmt} ${BASE_miss} ${BASE_cover}
+
+ """ >> ${output_file} + if [[ ${coverage_status} = "SUCCESS" ]]; then + echo """""" >> ${output_file} + echo "coverage PASS, no need to compare difference" + exit 0 + fi + echo """ +
+

Coverage Detail

+ + + + + + + + + """ >> ${output_file} + # generate compare detail + cat $file_name | while read line + do + if [[ $(echo $line | grep "[0-9]a[0-9]") ]] && [[ $(grep -A 1 "$line" $file_name | grep ">") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep ">") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po ">.*[a-z,A-Z].*.py" | sed "s|>||g") + for diff_file in ${diff_file_name} + do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + file=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $1}') + miss=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $3}') + cover=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $6}') + branch=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $4}') + echo """ + + + + + """ >> ${output_file} + done + elif [[ $(echo $line | grep "[0-9]c[0-9]") ]] && [[ $(cat $file_name | grep -A 1 "$line" | grep "<") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g") + #diff_file_name=$(echo ${diff_lines} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|,||g) + for diff_file in ${diff_file_name} + do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + file1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $1}') + miss1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $3}') + cover1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $6}') + branch1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $4}') + file2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $1}') + miss2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $3}') + cover2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $6}') + branch2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $4}') + # if branch coverage not change, not consider as regression + [[ "${branch1}" == "${branch2}" ]] && continue + echo """ + + + + + """ >> ${output_file} + done + elif [[ $(echo $line | grep "[0-9]d[0-9]") ]] && [[ $(cat $file_name | grep -A 1 "$line" | grep "<") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g") + for diff_file in ${diff_file_name} + do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + file=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $1}') + miss=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $3}') + cover=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $6}') + branch=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $4}') + echo """ + + + + + """ >> ${output_file} + done + fi + done + # generage table end + echo """
CommitFileNameMissBranchCover
PR | BASE${file}NA | ${miss}NA | ${branch}NA | ${cover}
PR | BASE${file1}${miss1} | ${miss2}${branch1} | ${branch2}${cover1} | ${cover2}
PR | BASE${file} | NA${miss} | NA${branch} | NA${cover} | NA
""" >> ${output_file} + +} + +main diff --git a/.azure-pipelines/scripts/ut/coverage.file b/.azure-pipelines/scripts/ut/coverage.file new file mode 100644 index 00000000000..4e649650502 --- /dev/null +++ b/.azure-pipelines/scripts/ut/coverage.file @@ -0,0 +1,25 @@ +[run] +branch = True +relative_files=True +[paths] +source = + /usr/local/lib/python3.8/dist-packages/neural_compressor + /neural-compressor/neural_compressor +[report] +omit = + */**/fake*yaml + */**/fake.py + */neural_compressor/ux/* + */neural_compressor/model/nets_factory.py + */neural_compressor/experimental/benchmark.py + */neural_compressor/contrib/strategy/tpe.py + */nlp_toolkit/backends/* +exclude_lines = + pragma: no cover + raise NotImplementedError + raise TypeError + if self.device == "gpu": + if device == "gpu": + except ImportError: + onnx_version < ONNX18_VERSION + onnx_version >= ONNX18_VERSION diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh new file mode 100644 index 00000000000..096198ad3d5 --- /dev/null +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -x + +echo "tensorflow version is $tensorflow_version" +echo "pytorch version is $pytorch_version" +echo "torchvision version is $torchvision_version" +echo "ipex version is $ipex_version" +echo "onnx version is $onnx_version" +echo "onnxruntime version is $onnxruntime_version" +echo "mxnet version is $mxnet_version" + +if [[ "${tensorflow_version}" == *"-official" ]]; then + pip install tensorflow==${tensorflow_version%-official} +elif [[ "${tensorflow_version}" == "spr-base" ]]; then + pip install /tf_dataset/tf_binary/tensorflow*.whl + if [[ $? -ne 0 ]]; then + exit 1 + fi +elif [[ "${tensorflow_version}" != "" ]]; then + pip install intel-tensorflow==${tensorflow_version} +fi + +if [[ "${pytorch_version}" != "" ]]; then + pip install torch==${pytorch_version} -f https://download.pytorch.org/whl/torch_stable.html +fi + +if [[ "${torchvision_version}" != "" ]]; then + pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html +fi + +if [[ "${ipex_version}" != "" ]]; then + ipex_whl="http://intel-optimized-pytorch.s3.cn-north-1.amazonaws.com.cn/wheels/v1.12.0/intel_extension_for_pytorch-1.12.0%2Bcpu-cp38-cp38-linux_x86_64.whl" + pip install $ipex_whl +fi + +if [[ "${onnx_version}" != "" ]]; then + pip install onnx==${onnx_version} +fi + +if [[ "${onnxruntime_version}" != "" ]]; then + pip install onnxruntime==${onnxruntime_version} + pip install onnxruntime-extensions +fi + +if [ "${mxnet_version}" == '1.6.0' ]; then + pip install mxnet-mkl==${mxnet_version} +elif [ "${mxnet_version}" == '1.7.0' ]; then + pip install mxnet==${mxnet_version}.post2 +elif [ "${mxnet_version}" != '' ]; then + pip install mxnet==${mxnet_version} +fi + + +cd /neural-compressor/test +if [ -f "requirements.txt" ]; then + sed -i '/^neural-compressor/d' requirements.txt + sed -i '/^intel-tensorflow/d' requirements.txt + sed -i '/find-links https:\/\/download.pytorch.org\/whl\/torch_stable.html/d' requirements.txt + sed -i '/^torch/d;/^torchvision/d;/^intel-extension-for-pytorch/d' requirements.txt + sed -i '/^mxnet-mkl/d' requirements.txt + sed -i '/^onnx/d;/^onnxruntime/d;/^onnxruntime-extensions/d' requirements.txt + n=0 + until [ "$n" -ge 3 ] + do + python -m pip install --no-cache-dir -r requirements.txt && break + n=$((n+1)) + sleep 5 + done + pip list +else + echo "Not found requirements.txt file." +fi + +pip install coverage +pip install pytest diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh new file mode 100644 index 00000000000..6600029b1ad --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -x + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.12.0+cpu' +export torchvision_version='0.13.0+cpu' +export onnx_version='1.11.0' +export onnxruntime_version='1.11.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') +cd /neural-compressor/test || exit 1 +find ./adaptor -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} +cp .coverage ${LOG_DIR}/.coverage.adaptor +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh new file mode 100644 index 00000000000..2b687e633d3 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic adaptor tfnewapi" + +echo "specify fwk version..." +export tensorflow_version='spr-base' +# export FORCE_BF16=1 + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') +cd /neural-compressor/test || exit 1 +find ./tfnewapi -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_newapi.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} +cp .coverage ${LOG_DIR}/.coverage.tfnewapi +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_ipex.sh b/.azure-pipelines/scripts/ut/run_basic_ipex.sh new file mode 100644 index 00000000000..c5742a75e8c --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_ipex.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic adaptor tfnewapi" + +echo "specify fwk version..." +export pytorch_version='1.12.0+cpu' +export torchvision_version='0.13.0+cpu' +export ipex_version='1.12.0+cpu' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') +cd /neural-compressor/test || exit 1 +find ./ipex -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_ipex.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} +cp .coverage ${LOG_DIR}/.coverage.ipex +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh new file mode 100644 index 00000000000..0e5e91496e9 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic" + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.12.0+cpu' +export torchvision_version='0.13.0+cpu' +export onnx_version='1.11.0' +export onnxruntime_version='1.11.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') +echo "copy pre-train model..." +mkdir -p /tmp/.neural_compressor/inc_ut || true +cp -r /tf_dataset/ut-localfile/resnet_v2 /tmp/.neural_compressor/inc_ut || true + +cd /neural-compressor/test || exit 1 +find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh +sed -i '/ adaptor\//d' run.sh +sed -i '/ tfnewapi\//d' run.sh +sed -i '/ ux\//d' run.sh +sed -i '/ neural_coder\//d' run.sh +sed -i '/ ipex\//d' run.sh +sed -i '/ pruning\//d' run.sh +sed -i '/ distillation\//d' run.sh +sed -i '/ nas\//d' run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} +cp .coverage ${LOG_DIR}/.coverage.others +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ncoder.sh b/.azure-pipelines/scripts/ut/run_ncoder.sh new file mode 100644 index 00000000000..aef05d13e3f --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_ncoder.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run coder" + +echo "no FWKs need to be installed..." +echo "no requirements need to be installed..." + +cd /neural-compressor/test || exit 1 +find ./neural_coder -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_neural_coder.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ux.sh b/.azure-pipelines/scripts/ut/run_ux.sh new file mode 100644 index 00000000000..ddfff991652 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_ux.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run ux" + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./ux -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/template/code-scan-template.yml b/.azure-pipelines/template/code-scan-template.yml new file mode 100644 index 00000000000..d0a62a2cef6 --- /dev/null +++ b/.azure-pipelines/template/code-scan-template.yml @@ -0,0 +1,30 @@ +parameters: +- name: codeScanFileName + type: string +- name: uploadPath + type: string + +- name: codeScanContainerName + type: string + default: 'codeScan' + +steps: + - template: docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'code-scan' + repoTag: '1.0' + dockerFileName: 'DockerfileCodeScan' + containerName: ${{ parameters.codeScanContainerName }} + + - script: | + docker exec ${{ parameters.codeScanContainerName }} bash /neural-compressor/.azure-pipelines/scripts/codeScan/${{ parameters.codeScanFileName }}/${{ parameters.codeScanFileName }}.sh + displayName: '${{ parameters.codeScanFileName }} Check' + + - task: PublishPipelineArtifact@1 + condition: always() + inputs: + targetPath: .azure-pipelines/scripts/codeScan/scanLog/${{ parameters.uploadPath }} + artifact: ${{ parameters.codeScanFileName }} + publishLocation: 'pipeline' + displayName: "PublishPipelineArtifact" \ No newline at end of file diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml new file mode 100644 index 00000000000..d3b25d3bcf8 --- /dev/null +++ b/.azure-pipelines/template/docker-template.yml @@ -0,0 +1,71 @@ +parameters: +- name: dockerConfigName + type: string + default: 'commonDockerConfig' +- name: repoName + type: string + default: 'neural-compressor' +- name: repoTag + type: string + default: 'py38' +- name: dockerFileName + type: string + default: 'Dockerfile' +- name: containerName + type: string +- name: repo + type: string + default: 'https://github.com/intel/neural-compressor' + +steps: +- ${{ if eq(parameters.dockerConfigName, 'commonDockerConfig') }}: + - script: | + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: 'Clean workspace' + + - checkout: self + clean: true + displayName: 'Checkout out Repo' + +- ${{ if eq(parameters.dockerConfigName, 'gitCloneDockerConfig') }}: + - script: | + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo mkdir ${BUILD_SOURCESDIRECTORY} + sudo chmod 777 ${BUILD_SOURCESDIRECTORY} + displayName: 'Clean workspace' + + - checkout: none + + - script: | + git clone ${{ parameters.repo }} ${BUILD_SOURCESDIRECTORY} + git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git checkout master + displayName: "Checkout out master" + +- script: | + if [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} . + fi + docker images | grep -i ${{ parameters.repoName }} + if [[ $? -ne 0 ]]; then + echo "NO Such Repo" + exit 1 + fi + displayName: "Build develop docker image" + +- script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker container' + +- ${{ if ne(parameters.containerName, '') }}: + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run -dit --disable-content-trust --privileged --name=${{ parameters.containerName }} --shm-size="2g" \ + -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 ${{ parameters.repoName }}:${{ parameters.repoTag }} + displayName: 'Docker run - ${{ parameters.containerName }}Container' \ No newline at end of file diff --git a/.azure-pipelines/template/model-template.yml b/.azure-pipelines/template/model-template.yml new file mode 100644 index 00000000000..f145025faf2 --- /dev/null +++ b/.azure-pipelines/template/model-template.yml @@ -0,0 +1,76 @@ +parameters: + - name: modelName + type: string + default: "resnet50v1.5" + - name: framework + type: string + default: "tensorflow" + + - name: modelContainerName + type: string + default: "model" + +steps: + - template: docker-template.yml + parameters: + dockerConfigName: "commonDockerConfig" + repoName: "neural-compressor" + repoTag: "py38" + dockerFileName: "Dockerfile" + containerName: ${{ parameters.modelContainerName }} + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='env_setup'" + displayName: Env setup + + - task: DownloadPipelineArtifact@2 + inputs: + source: "specific" + artifact: ${{ parameters.framework }}_${{ parameters.modelName }} + patterns: "**_summary.log" + path: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}_refer_log + project: $(System.TeamProject) + pipeline: "Model-Test" + runVersion: "specific" + runId: $(refer_buildId) + retryDownloadCount: 3 + displayName: "Download refer logs" + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='tuning'" + displayName: Tuning + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='int8_benchmark'" + displayName: INT8 Benchmark + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='fp32_benchmark'" + displayName: FP32 Benchmark + + - task: Bash@3 + condition: always() + inputs: + targetType: "inline" + script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='collect_log'" + displayName: Collect log + + - task: PublishPipelineArtifact@1 + condition: always() + inputs: + targetPath: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}/ + artifact: ${{ parameters.framework }}_${{ parameters.modelName }} + publishLocation: "pipeline" + + - script: | + if [ ${{ parameters.framework }}_${{ parameters.modelName }}_failed == 'true' ]; then + echo "[Failed] Model ${{ parameters.modelName }} failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" diff --git a/.azure-pipelines/template/ut-basic-template.yml b/.azure-pipelines/template/ut-basic-template.yml new file mode 100644 index 00000000000..ea1a00f73e6 --- /dev/null +++ b/.azure-pipelines/template/ut-basic-template.yml @@ -0,0 +1,39 @@ +parameters: +- name: dockerConfigName + type: string + default: 'commonDockerConfig' +- name: repo + type: string + default: 'https://github.com/intel/neural-compressor' +- name: utScriptFileName + type: string +- name: uploadPath + type: string +- name: utArtifact + type: string + +- name: utContainerName + type: string + default: 'utBasic' + +steps: + - template: docker-template.yml + parameters: + dockerConfigName: ${{ parameters.dockerConfigName }} + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: ${{ parameters.utContainerName }} + repo: ${{ parameters.repo }} + + - script: | + docker exec ${{ parameters.utContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/${{ parameters.utScriptFileName }}.sh" + displayName: 'Install Neural Compressor and run UT' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: ${{ parameters.uploadPath }} + artifact: ${{ parameters.utArtifact }} + publishLocation: 'pipeline' \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml new file mode 100644 index 00000000000..9d49fab17eb --- /dev/null +++ b/.azure-pipelines/ut-basic.yml @@ -0,0 +1,167 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_compressor + exclude: + - neural_compressor/ux + +pool: ICX-16C + +variables: + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir + DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir + ARTIFACT_NAME: 'UT_coverage_report' + REPO: 'https://github.com/intel/neural-compressor' + + +stages: +- stage: Adaptor + displayName: Unit Test FWKs adaptor + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_adaptor' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-adaptor' + +- stage: TFNewAPI + displayName: Unit Test tf newAPI + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_adaptor_tfnewapi' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-tfnewapi' + +- stage: IPEX + displayName: Unit Test IPEX + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_ipex' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-ipex' + +- stage: Others + displayName: Unit Test other basic case + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_others' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-others' + +- stage: Adaptor_base + displayName: Unit Test FWKs adaptor baseline + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_adaptor' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-adaptor-base' + repo: $(REPO) + +- stage: TFNewAPI_base + displayName: Unit Test tf newAPI baseline + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_adaptor_tfnewapi' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-tfnewapi-base' + repo: $(REPO) + +- stage: IPEX_base + displayName: Unit Test IPEX baseline + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_ipex' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-ipex-base' + repo: $(REPO) + +- stage: Others_base + displayName: Unit Test other cases baseline + dependsOn: [] + jobs: + - job: + steps: + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_others' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-others-base' + repo: $(REPO) + +- stage: Coverage + displayName: "Coverage Combine" + dependsOn: [Adaptor, TFNewAPI, Others, IPEX, Adaptor_base, TFNewAPI_base, Others_base, IPEX_base] + jobs: + - job: CollectDatafiles + steps: + - template: template/docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: '' + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + path: $(DOWNLOAD_PATH) + + - script: | + echo "--- create container ---" + docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash + echo "--- docker ps ---" + docker ps + echo "--- collect logs ---" + docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/collect_log.sh" + displayName: 'collect logs' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: $(ARTIFACT_NAME) + publishLocation: 'pipeline' \ No newline at end of file diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml new file mode 100644 index 00000000000..ba8bc094e46 --- /dev/null +++ b/.azure-pipelines/ut-ncoder.yml @@ -0,0 +1,34 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_coder + +pool: ICX-16C + +variables: + NCODER_CONTAINER_NAME: 'nCoderContainer' + +stages: +- stage: + displayName: Unit Test for Neural Coder + jobs: + - job: + steps: + - template: template/docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: $(NCODER_CONTAINER_NAME) + + - script: | + docker exec $(NCODER_CONTAINER_NAME) bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ncoder.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml new file mode 100644 index 00000000000..386727e8d09 --- /dev/null +++ b/.azure-pipelines/ut-ux.yml @@ -0,0 +1,34 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_compressor/ux + +pool: ICX-16C + +variables: + UX_CONTAINER_NAME: 'UXContainer' + +stages: +- stage: + displayName: Unit Test for UX + jobs: + - job: + steps: + - template: template/docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: $(UX_CONTAINER_NAME) + + - script: | + docker exec $(UX_CONTAINER_NAME) bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ux.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file