Wenxin/ut coverage

.azure-pipelines/azure-pipelines-demo.yml

@@ -0,0 +1,51 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger: none
+pr:
+- wenxin-test
+
+pool: docker-agent
+
+variables:
+  IMAGE_NAME: 'neural-compressor'
+  IMAGE_TAG: 'py38'
+
+stages:
+- stage: Build
+  displayName: Build and push stage
+  jobs:
+  - job: Build
+    displayName: Build
+    steps:
+    - script: |
+        echo ${BUILD_SOURCESDIRECTORY}
+        sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
+      displayName: 'Clean workspace'
+
+    - checkout: self
+      displayName: "Checkout out Repo"
+
+    - script: |
+        if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
+            docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
+        fi
+        docker images | grep -i ${IMAGE_NAME}
+        if [[ $? -ne 0 ]]; then
+          echo "NO Such Image ${IMAGE_NAME}"
+          exit 1
+        fi
+      displayName: "Build Devel Images"
+
+    - script: |
+        docker stop $(docker ps -aq)
+        docker rm -vf $(docker ps -aq) || true
+      displayName: 'Clean Docker'
+
+    - script: |
+        docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor  ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list  && pip install intel-tensorflow &&  python test/strategy/test_bayesian.py"
+      displayName: 'Build Binary and Run UT'
+
+

.azure-pipelines/code-scan.yml

@@ -0,0 +1,210 @@
+trigger: none
+pr: 
+  - master
+pool: suyue-test
+
+variables:
+  IMAGE_NAME: "code-scan"
+  IMAGE_TAG: "1.0"
+  CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan"
+  CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog"
+  CURRENT_PATH: $(Build.SourcesDirectory)
+  CURRENT_STATUS: true
+  ER: "1212"
+
+stages:
+  - stage: BanditCodeScan
+    displayName: Bandit Code Scan
+    dependsOn: []
+    jobs:
+      - job: Bandit
+        displayName: Bandit
+        condition: eq(variables['CURRENT_STATUS'], 'true')
+        steps:
+          - script: |
+              echo ${BUILD_SOURCESDIRECTORY}
+              sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
+              echo y | docker system prune
+            displayName: "Clean workspace"
+          - checkout: self
+            displayName: "Checkout out Repo"
+          - script: |
+              if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
+                  docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
+              fi
+              docker images | grep -i ${IMAGE_NAME}
+              if [[ $? -ne 0 ]]; then
+                echo "NO Such Image ${IMAGE_NAME}"
+                exit 1
+              fi
+            displayName: "Build Devel Images"
+          - script: |
+              docker stop $(docker ps -aq)
+              docker rm -vf $(docker ps -aq) || true
+            displayName: "Clean Docker"
+
+          - task: Bash@3
+            inputs:
+              targetType: "inline"
+              script: |
+                docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS)
+                echo "------$(CURRENT_STATUS)-------"
+                echo $(CURRENT_STATUS)
+
+                echo "---$(ER)----$(QR)------"
+          - task: PublishPipelineArtifact@1
+            inputs:
+              targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log
+              artifact: bandit
+              publishLocation: "pipeline"
+
+          # - script: |
+          #     if [ $(CURRENT_STATUS) -ne 0 ] ; then
+          #         exit 1
+          #     fi
+          #     exit 0
+
+
+
+  # - stage: PylintCodeScan
+  #   displayName: Pylint Code Scan
+  #   dependsOn: []
+  #   jobs:
+  #     - job: Pylint
+  #       displayName: Pylint
+  #       steps:
+  #         - script: |
+  #             if [ $(CURRENT_STATUS) -ne 0 ] ; then
+  #                 exit 1
+  #             fi
+  #             exit 0
+  #         - script: |
+  #             echo ${BUILD_SOURCESDIRECTORY}
+  #             sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
+  #             echo y | docker system prune
+  #           displayName: "Clean workspace"
+  #         - checkout: self
+  #           displayName: "Checkout out Repo"
+  #         - script: |
+  #             if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
+  #                 docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
+  #             fi
+  #             docker images | grep -i ${IMAGE_NAME}
+  #             if [[ $? -ne 0 ]]; then
+  #               echo "NO Such Image ${IMAGE_NAME}"
+  #               exit 1
+  #             fi
+  #           displayName: "Build Devel Images"
+  #         - script: |
+  #             docker stop $(docker ps -aq)
+  #             docker rm -vf $(docker ps -aq) || true
+  #           displayName: "Clean Docker"
+
+  #         - task: Bash@3
+  #           inputs:
+  #             targetType: "inline"
+  #             script: |
+  #               docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh
+
+  #         - task: PublishPipelineArtifact@1
+  #           inputs:
+  #             targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json
+  #             artifact: pylint
+  #             publishLocation: "pipeline"
+
+  #         - script: |
+  #             if [ $(CURRENT_STATUS) -ne 0 ] ; then
+  #                 exit 1
+  #             fi
+  #             exit 0
+
+  # - stage: PyspellingCodeScan
+  #   displayName: Pyspelling Code Scan
+  #   dependsOn: []
+  #   jobs:
+  #     - job: Pyspelling
+  #       displayName: Pyspelling
+  #       steps:
+  #         - script: |
+  #             if [ $(CURRENT_STATUS) -ne 0 ] ; then
+  #                 exit 1
+  #             fi
+  #             exit 0
+  #         - script: |
+  #             echo ${BUILD_SOURCESDIRECTORY}
+  #             sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true
+  #             echo y | docker system prune
+  #           displayName: "Clean workspace"
+  #         - checkout: self
+  #           displayName: "Checkout out Repo"
+  #         - script: |
+  #             if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then
+  #                 docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
+  #             fi
+  #             docker images | grep -i ${IMAGE_NAME}
+  #             if [[ $? -ne 0 ]]; then
+  #               echo "NO Such Image ${IMAGE_NAME}"
+  #               exit 1
+  #             fi
+  #           displayName: "Build Devel Images"
+  #         - script: |
+  #             docker stop $(docker ps -aq)
+  #             docker rm -vf $(docker ps -aq) || true
+  #           displayName: "Clean Docker"
+
+  #         - task: Bash@3
+  #           inputs:
+  #             targetType: "inline"
+  #             script: |
+  #               docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh
+
+  #         - task: PublishPipelineArtifact@1
+  #           inputs:
+  #             targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log
+  #             artifact: pyspelling
+  #             publishLocation: "pipeline"              
+
+  # - stage: CopyRight
+  #   displayName: CopyRight Code Scan
+  #   dependsOn: []
+  #   jobs:
+  #     - job: CopyRight
+  #       displayName: CopyRight
+  #       steps:
+  #         - task: Bash@3
+  #           inputs:
+  #             targetType: "inline"
+  #             script: |
+  #               set -ex
+  #               supported_extensions=(py, sh, yaml)
+
+  #               git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log
+  #               files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++')
+
+  #               for file in ${files}
+  #               do
+  #                   if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then
+  #                       echo "Checking license in 1 ${file}"
+  #                       if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then
+  #                           echo ${file} >>  $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
+  #                       fi
+  #                   else
+  #                       echo "Skipping ${file}"
+  #                   fi
+  #               done
+
+  #         - task: PublishPipelineArtifact@1
+  #           inputs:
+  #             targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
+  #             artifact: copyright
+  #             publishLocation: "pipeline" 
+
+  #         - task: Bash@3
+  #           inputs:
+  #             targetType: "inline"
+  #             script: |
+  #               ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log
+  #               if [ $? -eq 0 ] ; then
+  #                   echo "------------------Check <copyright_issue_summary.log> for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1
+  #               fi
+  #               exit 0

.azure-pipelines/docker/Dockerfile.devel

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG UBUNTU_VER=20.04
+FROM ubuntu:${UBUNTU_VER} as devel
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+    python3 \
+    python3-pip \
+    python3-dev \
+    python3-distutils \
+    autoconf \
+    build-essential \
+    git \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    numactl \
+    time \
+    wget
+
+RUN ln -sf $(which python3) /usr/bin/python
+
+RUN python -m pip --no-cache-dir install --upgrade pip
+RUN python -m pip install --no-cache-dir setuptools
+
+RUN pip list
+
+WORKDIR /
+

.azure-pipelines/docker/DockerfileWithNC.devel

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2022 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG UBUNTU_VER=20.04
+FROM ubuntu:${UBUNTU_VER} as devel
+
+# See http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
+ARG REPO=x
+ARG TARGET_BRANCH=y
+ARG SOURCE_BRANCH=z
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
+    python3 \
+    python3-pip \
+    python3-dev \
+    python3-distutils \
+    autoconf \
+    build-essential \
+    git \
+    libgl1-mesa-glx \
+    libglib2.0-0
+
+RUN ln -sf $(which python3) /usr/bin/python
+
+RUN python -m pip --no-cache-dir install --upgrade pip
+
+RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \
+    cd neural-compressor && \
+    python -m pip install --no-cache-dir setuptools && \
+    python -m pip install --no-cache-dir pycocotools && \
+    python -m pip install --no-cache-dir -r requirements.txt && \
+    python setup.py install
+
+RUN pip list
+RUN pip list | grep neural
+
+WORKDIR /neural-compressor
+RUN git rev-parse HEAD

.azure-pipelines/scripts/codeScan/bandit/bandit.sh

@@ -0,0 +1,16 @@
+# set -ex
+bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog"
+pip install bandit
+
+python -m bandit -r -lll -iii /neural-compressor/neural_compressor >  $bandit_log_dir/lpot-bandit.log
+
+exit_code=$?
+if [ ${exit_code} -eq 0 ] ; then
+    sed -i "s|CURRENT_STATUS\: true|CURRENT_STATUS\: false|g" /neural-compressor/azure-pipelines.yml
+    sed -i "s|ER|QR|g" /neural-compressor/azure-pipelines.yml
+
+    # sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml
+    echo "Bandit exited with non-zero exit code."; exit 1
+fi
+exit 0
+