From db2f5fc0c9f4f795a11c1accf25062b36e1c98d8 Mon Sep 17 00:00:00 2001 From: letonghan <106566639+letonghan@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:11:33 +0800 Subject: [PATCH 001/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000000..6f401c61650 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,36 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +pool: + vmImage: ubuntu-latest +strategy: + matrix: + Python27: + python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From 02819667f2c4ad32cee585885286f259796f5b48 Mon Sep 17 00:00:00 2001 From: letonghan <106566639+letonghan@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:45:07 +0800 Subject: [PATCH 002/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 93 ++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6f401c61650..7530cb8974d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,36 +1,67 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml -trigger: -- master +pr: + - main -pool: - vmImage: ubuntu-latest -strategy: - matrix: - Python27: - python.version: '2.7' - Python35: - python.version: '3.5' - Python36: - python.version: '3.6' - Python37: - python.version: '3.7' +pool: wenjiao-test -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' +stages: + - stage: pylint + jobs: + - job: pylint_test + steps: + - task: Bash@3 + inputs: + targetType: "inline" + script: | + pwd + pip install pylint==2.12.1 + pip install intel-tensorflow + python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./neural_compressor > /lpot-pylint.json + # # exit_code=$? + # # if [ ${exit_code} -ne 0 ] ; then + # # echo "PyLint exited with non-zero exit code."; exit 1 + # # fi + # # exit 0 -- script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' + - stage: bandit + jobs: + - job: bandit_test + steps: + - task: Bash@3 + inputs: + targetType: "inline" +# script: | +# pip install bandit +# # python -m bandit -r -lll -iii neural_compressor > ${WORKSPACE}/lpot-bandit.log +# # exit_code=$? +# # if [ ${exit_code} -ne 0 ] ; then +# # echo "Bandit exited with non-zero exit code."; exit 1 +# # fi +# # exit 0 -- script: | - pip install pytest pytest-azurepipelines - pytest - displayName: 'pytest' + # - stage: pyspelling + # jobs: + # - job: pyspelling_test +# steps: +# - task: Bash@3 +# inputs: +# targetType: "inline" +# script: | +# pip install pyspelling +# # Update paths to validation and lpot repositories +# # VAL_REPO=${WORKSPACE} + +# # sed -i "s|\${VAL_REPO}|$VAL_REPO|g" ${VAL_REPO}/pyspelling_conf.yaml +# # sed -i "s|\${LPOT_REPO}|$REPO_DIR|g" ${VAL_REPO}/pyspelling_conf.yaml +# # echo "Modified config:" +# # cat ${VAL_REPO}/pyspelling_conf.yaml +# # pyspelling -c ${VAL_REPO}/pyspelling_conf.yaml > ${WORKSPACE}/pyspelling_output.log +# # exit_code=$? +# # if [ ${exit_code} -ne 0 ] ; then +# # echo "Pyspelling exited with non-zero exit code."; exit 1 +# # fi +# # exit 0 From ecedb0a3a8d3dd764aa3941c0630de81b37ff02f Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 14:32:27 +0800 Subject: [PATCH 003/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines-1.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 azure-pipelines-1.yml diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml new file mode 100644 index 00000000000..b384d61c2fb --- /dev/null +++ b/azure-pipelines-1.yml @@ -0,0 +1,40 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +pr: +- wenxin-test + +pool: + vmImage: ubuntu-latest + +variables: + PYTHON: 'python3.8' + INC_VER: '1.12' + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: '1.12' + INC_BRANCH: 'master' + +stages: +- stage: Build + displayName: Build and push stage + jobs: + - job: Build + displayName: Build + steps: + - script: docker build --build-arg PYTHON=${PYTHON} --build-arg INC_BRANCH=${INC_BRANCH} -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + displayName: 'Build Container' + + - script: | + docker images | grep -i neural-compressor + displayName: 'Check docker' + + - script: | + Workspace=`pwd` + docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null + docker exec test-inc python3 -c "import neural_compressor as nc;print(nc.version.__version__)" + docker exec test-inc python3.8 -c "import neural_compressor as nc;print(nc.version.__version__)" + docker exec test-inc pwd && ls -l && pip list && python test/test_metrics.py + + displayName: 'Run python sciprt' From e2e9e01cb9817021a59e011e2d910ccd3d840dd7 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 14:42:47 +0800 Subject: [PATCH 004/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index b384d61c2fb..788bc4c4d6f 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -33,8 +33,8 @@ stages: - script: | Workspace=`pwd` docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null - docker exec test-inc python3 -c "import neural_compressor as nc;print(nc.version.__version__)" - docker exec test-inc python3.8 -c "import neural_compressor as nc;print(nc.version.__version__)" + docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" + docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" docker exec test-inc pwd && ls -l && pip list && python test/test_metrics.py displayName: 'Run python sciprt' From 6635449cb58c0054b36f2930d2ed418485c467bd Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 14:51:29 +0800 Subject: [PATCH 005/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 788bc4c4d6f..42fb8a94795 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -10,7 +10,7 @@ pool: vmImage: ubuntu-latest variables: - PYTHON: 'python3.8' + PYTHON: 'python' INC_VER: '1.12' IMAGE_NAME: 'neural-compressor' IMAGE_TAG: '1.12' @@ -32,9 +32,10 @@ stages: - script: | Workspace=`pwd` + docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker exec test-inc pwd && ls -l && pip list && python test/test_metrics.py + docker exec test-inc pip install numpy && pip list && python test/test_metrics.py displayName: 'Run python sciprt' From 37cc4c4de821812a3d4fc1483903679cb3b7a5a8 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 14:53:34 +0800 Subject: [PATCH 006/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 42fb8a94795..f69a0649ed0 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -10,7 +10,7 @@ pool: vmImage: ubuntu-latest variables: - PYTHON: 'python' + PYTHON: 'python3.8' INC_VER: '1.12' IMAGE_NAME: 'neural-compressor' IMAGE_TAG: '1.12' From 92ff7200ba934ca625dbc351e2f9f04b3662e801 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 14:58:20 +0800 Subject: [PATCH 007/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index f69a0649ed0..3609cc94963 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -33,6 +33,7 @@ stages: - script: | Workspace=`pwd` docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" + docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python test/test_metrics.py docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" From 81f28ee9af7eef957f9fa3d668ce09ec2a665a17 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 15:21:58 +0800 Subject: [PATCH 008/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 3609cc94963..95c3c4a08af 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -32,8 +32,8 @@ stages: - script: | Workspace=`pwd` - docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python test/test_metrics.py + docker run -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" + docker run -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/test_metrics.py docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" From f3b5d2b6dea51b4da4087b934fe61f4503aaa3dd Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 15:28:40 +0800 Subject: [PATCH 009/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 95c3c4a08af..1be74fcdb9d 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -32,8 +32,8 @@ stages: - script: | Workspace=`pwd` - docker run -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker run -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/test_metrics.py + docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" + docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/test_metrics.py docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" From 931b25196afbf6e723a26b957d01e4c6eaf8ba63 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 15:44:38 +0800 Subject: [PATCH 010/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 1be74fcdb9d..50c31f2bd52 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -32,11 +32,12 @@ stages: - script: | Workspace=`pwd` + docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -c "pwd && ls -l && pip list" docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/test_metrics.py + docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/config/test_config.py docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker exec test-inc pip install numpy && pip list && python test/test_metrics.py + docker exec test-inc /usr/local/bin/python test/test_metrics.py displayName: 'Run python sciprt' From 0dc83f9ebd153c74b1048bea75926ab41a1cb62c Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 15:55:05 +0800 Subject: [PATCH 011/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 50c31f2bd52..59c944a15c2 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -34,10 +34,8 @@ stages: Workspace=`pwd` docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -c "pwd && ls -l && pip list" docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/config/test_config.py + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/config/test_config.py docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" - docker exec test-inc /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker exec test-inc /usr/local/bin/python test/test_metrics.py displayName: 'Run python sciprt' From a83a2481abec6d3d980d8df606b92a9737fb3c55 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 28 Jul 2022 16:01:03 +0800 Subject: [PATCH 012/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 59c944a15c2..af56bf8e184 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -31,11 +31,7 @@ stages: displayName: 'Check docker' - script: | - Workspace=`pwd` - docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -c "pwd && ls -l && pip list" docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/config/test_config.py - docker run --init --name=test-inc -d -u root:root -w ${Workspace} ${IMAGE_NAME}:${IMAGE_TAG} tail -f /dev/null - docker exec test-inc /usr/local/bin/python3 -c "import neural_compressor as nc;print(nc.version.__version__)" displayName: 'Run python sciprt' From 1fbd878160d6a9c4f1003a9b0d67f9bb57490ab8 Mon Sep 17 00:00:00 2001 From: letonghan <106566639+letonghan@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:11:33 +0800 Subject: [PATCH 013/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000000..6f401c61650 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,36 @@ +# Python package +# Create and test a Python package on multiple Python versions. +# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/python + +trigger: +- master + +pool: + vmImage: ubuntu-latest +strategy: + matrix: + Python27: + python.version: '2.7' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + +- script: | + python -m pip install --upgrade pip + pip install -r requirements.txt + displayName: 'Install dependencies' + +- script: | + pip install pytest pytest-azurepipelines + pytest + displayName: 'pytest' From 0518a4f98404a42ac7948c777d46e3e0d0a8e0f7 Mon Sep 17 00:00:00 2001 From: letonghan <106566639+letonghan@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:45:07 +0800 Subject: [PATCH 014/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines.yml | 93 ++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 31 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6f401c61650..7530cb8974d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,36 +1,67 @@ -# Python package -# Create and test a Python package on multiple Python versions. -# Add steps that analyze code, save the dist with the build record, publish to a PyPI-compatible index, and more: -# https://docs.microsoft.com/azure/devops/pipelines/languages/python +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml -trigger: -- master +pr: + - main -pool: - vmImage: ubuntu-latest -strategy: - matrix: - Python27: - python.version: '2.7' - Python35: - python.version: '3.5' - Python36: - python.version: '3.6' - Python37: - python.version: '3.7' +pool: wenjiao-test -steps: -- task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - displayName: 'Use Python $(python.version)' +stages: + - stage: pylint + jobs: + - job: pylint_test + steps: + - task: Bash@3 + inputs: + targetType: "inline" + script: | + pwd + pip install pylint==2.12.1 + pip install intel-tensorflow + python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./neural_compressor > /lpot-pylint.json + # # exit_code=$? + # # if [ ${exit_code} -ne 0 ] ; then + # # echo "PyLint exited with non-zero exit code."; exit 1 + # # fi + # # exit 0 -- script: | - python -m pip install --upgrade pip - pip install -r requirements.txt - displayName: 'Install dependencies' + - stage: bandit + jobs: + - job: bandit_test + steps: + - task: Bash@3 + inputs: + targetType: "inline" +# script: | +# pip install bandit +# # python -m bandit -r -lll -iii neural_compressor > ${WORKSPACE}/lpot-bandit.log +# # exit_code=$? +# # if [ ${exit_code} -ne 0 ] ; then +# # echo "Bandit exited with non-zero exit code."; exit 1 +# # fi +# # exit 0 -- script: | - pip install pytest pytest-azurepipelines - pytest - displayName: 'pytest' + # - stage: pyspelling + # jobs: + # - job: pyspelling_test +# steps: +# - task: Bash@3 +# inputs: +# targetType: "inline" +# script: | +# pip install pyspelling +# # Update paths to validation and lpot repositories +# # VAL_REPO=${WORKSPACE} + +# # sed -i "s|\${VAL_REPO}|$VAL_REPO|g" ${VAL_REPO}/pyspelling_conf.yaml +# # sed -i "s|\${LPOT_REPO}|$REPO_DIR|g" ${VAL_REPO}/pyspelling_conf.yaml +# # echo "Modified config:" +# # cat ${VAL_REPO}/pyspelling_conf.yaml +# # pyspelling -c ${VAL_REPO}/pyspelling_conf.yaml > ${WORKSPACE}/pyspelling_output.log +# # exit_code=$? +# # if [ ${exit_code} -ne 0 ] ; then +# # echo "Pyspelling exited with non-zero exit code."; exit 1 +# # fi +# # exit 0 From 1f428c93243d006863c588144624e7afba187a43 Mon Sep 17 00:00:00 2001 From: letonghan <106566639+letonghan@users.noreply.github.com> Date: Thu, 4 Aug 2022 10:49:05 +0800 Subject: [PATCH 015/296] create pipeline [skip ci] --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7530cb8974d..77719aa486b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,7 +4,7 @@ # https://aka.ms/yaml pr: - - main + - letong_performance pool: wenjiao-test From 6cd0f3430d1f1e64cf8389926b55f2e2ae90f42a Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Thu, 4 Aug 2022 14:01:41 +0800 Subject: [PATCH 016/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines-1.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 azure-pipelines-1.yml diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml new file mode 100644 index 00000000000..27a798ad834 --- /dev/null +++ b/azure-pipelines-1.yml @@ -0,0 +1,19 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: ubuntu-latest + +steps: +- script: echo Hello, world! + displayName: 'Run a one-line script' + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' From 4a2a86266ab30cddcafb4a4510cdccece6b199f6 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Thu, 4 Aug 2022 15:58:03 +0800 Subject: [PATCH 017/296] Set up CI with Azure Pipelines [skip ci] --- azure-pipelines-1.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index af56bf8e184..7704f650db5 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -6,8 +6,7 @@ pr: - wenxin-test -pool: - vmImage: ubuntu-latest +pool: suyue-test variables: PYTHON: 'python3.8' From 1bca1623f8752a611b7f7161ce46b48ae4197b1b Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 15:46:17 +0800 Subject: [PATCH 018/296] Update Dockerfile.devel --- docker/Dockerfile.devel | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index 52854413542..66d7d76c81c 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -46,13 +46,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ libglib2.0-0 -ARG INC_BRANCH=v1.12 -RUN git clone --single-branch --branch=${INC_BRANCH} https://github.com/intel/neural-compressor.git && \ - cd neural-compressor && \ - git submodule sync && \ - git submodule update --init --recursive && \ - python -m pip install --no-cache-dir pycocotools && \ - python -m pip install --no-cache-dir -r requirements.txt && \ - python setup.py install - +#ARG INC_BRANCH=v1.12 +#RUN git clone --single-branch --branch=${INC_BRANCH} https://github.com/intel/neural-compressor.git && \ +# cd neural-compressor && \ +# git submodule sync && \ +# git submodule update --init --recursive && \ +# python -m pip install --no-cache-dir pycocotools && \ +# python -m pip install --no-cache-dir -r requirements.txt && \ +# python setup.py install + +WORKDIR / +COPY ./ /neural_compressor +RUN pwd +RUN ls -l +RUN ${PYTHON} setup.py install WORKDIR /neural-compressor From f5d61841aa2977401d390625d885c0be3240fe03 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 17:07:20 +0800 Subject: [PATCH 019/296] Update Dockerfile.devel --- docker/Dockerfile.devel | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index 66d7d76c81c..11204ca98db 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -56,8 +56,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ # python setup.py install WORKDIR / -COPY ./ /neural_compressor -RUN pwd -RUN ls -l -RUN ${PYTHON} setup.py install -WORKDIR /neural-compressor +#COPY ./ /neural_compressor +#RUN pwd +#RUN ls -l +#RUN ${PYTHON} setup.py install +#WORKDIR /neural-compressor From 3af7637b7e22839e522741a688f89f5ca26ca4ea Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 17:13:14 +0800 Subject: [PATCH 020/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 7704f650db5..72fb9b777d4 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -10,10 +10,8 @@ pool: suyue-test variables: PYTHON: 'python3.8' - INC_VER: '1.12' IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: '1.12' - INC_BRANCH: 'master' + IMAGE_TAG: 'py38' stages: - stage: Build @@ -22,15 +20,10 @@ stages: - job: Build displayName: Build steps: - - script: docker build --build-arg PYTHON=${PYTHON} --build-arg INC_BRANCH=${INC_BRANCH} -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + - script: docker build --build-arg PYTHON=${PYTHON} -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . displayName: 'Build Container' - script: | docker images | grep -i neural-compressor displayName: 'Check docker' - - script: | - docker run ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python -c "import neural_compressor as nc;print(nc.version.__version__)" - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /usr/local/bin/python /neural-compressor/test/config/test_config.py - - displayName: 'Run python sciprt' From 9e054a84b27662f84163e05097ed48dad584e410 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 17:14:44 +0800 Subject: [PATCH 021/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 72fb9b777d4..1d56cb16da4 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -6,7 +6,7 @@ pr: - wenxin-test -pool: suyue-test +pool: docker-agent variables: PYTHON: 'python3.8' From e19e702d24308339c9dfc49217dc14b937b31599 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 18:45:36 +0800 Subject: [PATCH 022/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 1d56cb16da4..0ba880ebe07 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -20,8 +20,10 @@ stages: - job: Build displayName: Build steps: - - script: docker build --build-arg PYTHON=${PYTHON} -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - displayName: 'Build Container' + - script: | + pwd + ls -l + displayName: 'find repo' - script: | docker images | grep -i neural-compressor From b23d2bf769638bd56d322b7ad00e75e47ef07364 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 18:52:04 +0800 Subject: [PATCH 023/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 0ba880ebe07..e696823dc0f 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,6 +23,7 @@ stages: - script: | pwd ls -l + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python setup.py install" displayName: 'find repo' - script: | From a1eb428b29caed009e807fb4201de9293c3ff87f Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 18:54:55 +0800 Subject: [PATCH 024/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index e696823dc0f..603989fb69d 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python setup.py install" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" displayName: 'find repo' - script: | From 9a846e0acf8b73fe84b50c37c8694a86cfdd7141 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 19:17:19 +0800 Subject: [PATCH 025/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 603989fb69d..c7c95197e27 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" + docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" displayName: 'find repo' - script: | From 3273a9f0ec91633c7bde9761db440917fe57d4d7 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 19:21:08 +0800 Subject: [PATCH 026/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index c7c95197e27..fb729f22c63 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" + #docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" displayName: 'find repo' - script: | From 121d43c9b810ae3c3b718bcb614c137728b61bb3 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 19:24:52 +0800 Subject: [PATCH 027/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index fb729f22c63..603989fb69d 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - #docker run ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" displayName: 'find repo' - script: | From 5ad83d2d3987e3b6fdd7581628a4047ffd6c2864 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 19:35:25 +0800 Subject: [PATCH 028/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 603989fb69d..61675fc255d 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,10 +23,11 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list && python test/test_basic.py" displayName: 'find repo' - script: | docker images | grep -i neural-compressor + docker ps displayName: 'Check docker' From e9e76f0f014f12cb17eae78c08d5b17331ff6a4e Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Fri, 5 Aug 2022 19:39:42 +0800 Subject: [PATCH 029/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 61675fc255d..252d4d60056 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -21,13 +21,16 @@ stages: displayName: Build steps: - script: | - pwd - ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list && python test/test_basic.py" + pwd + ls -l + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list && python test/test_basic.py" displayName: 'find repo' - script: | docker images | grep -i neural-compressor docker ps displayName: 'Check docker' - + + - script: | + sudo rm -fr /home/azureuser-inc/myagent/workspace/1/s/* + displayName: 'clean up' From df26b3ef1a6eb450c5565020ffb3fde3607e47d7 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Sun, 7 Aug 2022 21:22:46 +0800 Subject: [PATCH 030/296] add azure pipeline for ut --- .azure-pipelines/docker/Dockerfile.devel | 53 +++++++++ .azure-pipelines/scripts/ut/run_basic.sh | 4 + .../scripts/ut/run_basic_adaptor.sh | 4 + .../scripts/ut/run_basic_adaptor_tfnewapi.sh | 4 + .azure-pipelines/scripts/ut/run_ncoder.sh | 4 + .azure-pipelines/scripts/ut/run_ux.sh | 4 + .azure-pipelines/ut-basic.yml | 104 ++++++++++++++++++ .azure-pipelines/ut-ncoder.yml | 44 ++++++++ .azure-pipelines/ut-ux.yml | 45 ++++++++ 9 files changed, 266 insertions(+) create mode 100644 .azure-pipelines/docker/Dockerfile.devel create mode 100644 .azure-pipelines/scripts/ut/run_basic.sh create mode 100644 .azure-pipelines/scripts/ut/run_basic_adaptor.sh create mode 100644 .azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh create mode 100644 .azure-pipelines/scripts/ut/run_ncoder.sh create mode 100644 .azure-pipelines/scripts/ut/run_ux.sh create mode 100644 .azure-pipelines/ut-basic.yml create mode 100644 .azure-pipelines/ut-ncoder.yml create mode 100644 .azure-pipelines/ut-ux.yml diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel new file mode 100644 index 00000000000..e7ef3909552 --- /dev/null +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -0,0 +1,53 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG UBUNTU_VER=20.04 +FROM ubuntu:${UBUNTU_VER} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +ARG PYTHON=python3.8 +ARG REPO=x +ARG TARGET_BRANCH=y +ARG SOURCE_BRANCH=z +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + python3 \ + python3-pip \ + ${PYTHON}-dev \ + ${PYTHON}-distutils \ + autoconf \ + build-essential \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 + +RUN ln -sf $(which ${PYTHON}) /usr/bin/python && \ + ln -sf $(which ${PYTHON}) /usr/bin/python3 + +RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \ + pip \ + setuptools + +RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ + cd neural-compressor && \ + python -m pip install --no-cache-dir pycocotools && \ + python -m pip install --no-cache-dir -r requirements.txt && \ + python setup.py install + +WORKDIR /neural-compressor +RUN git rev-parse HEAD diff --git a/.azure-pipelines/scripts/ut/run_basic.sh b/.azure-pipelines/scripts/ut/run_basic.sh new file mode 100644 index 00000000000..ec312e3a26e --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic" \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh new file mode 100644 index 00000000000..b1b4ea1df24 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic adaptor" diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh new file mode 100644 index 00000000000..fdcf2730401 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic adaptor tfnewapi" \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ncoder.sh b/.azure-pipelines/scripts/ut/run_ncoder.sh new file mode 100644 index 00000000000..9a13fbb4ce9 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_ncoder.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run coder" \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ux.sh b/.azure-pipelines/scripts/ut/run_ux.sh new file mode 100644 index 00000000000..d331242fa6b --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_ux.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run ux" \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml new file mode 100644 index 00000000000..16c3dc4109d --- /dev/null +++ b/.azure-pipelines/ut-basic.yml @@ -0,0 +1,104 @@ +trigger: none + +pr: + branches: + include: + - master + paths: + include: + - neural_compressor + exclude: + - neural_compressor.ux + +pool: suyue-test + +variables: + PYTHON: 'python3.8' + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'pr' + +stages: +- stage: + displayName: Unit Test FWKs adaptor + dependsOn: [] + jobs: + - job: + variables: + INC_BRANCH: $(System.PullRequest.SourceBranch) + REPO: $(Build.Repository.Uri) + steps: + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean Docker' + + - script: | + echo ${INC_BRANCH} + echo ${REPO} + docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + displayName: 'Build Image' + + - script: | + docker images | grep -i neural-compressor + displayName: 'Check docker image' + + - script: | + docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh + displayName: 'Run UT' + +- stage: + displayName: Unit Test tf newAPI + dependsOn: [] + jobs: + - job: + variables: + INC_BRANCH: $(System.PullRequest.SourceBranch) + REPO: $(Build.Repository.Uri) + steps: + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + echo ${INC_BRANCH} + echo ${REPO} + docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + displayName: 'Build Image' + + - script: | + docker images | grep -i neural-compressor + displayName: 'Check docker Image' + + - script: | + docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh + displayName: 'Run UT' + +- stage: + displayName: Unit Test other basic case + dependsOn: [] + jobs: + - job: + variables: + INC_BRANCH: $(System.PullRequest.SourceBranch) + REPO: $(Build.Repository.Uri) + steps: + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + echo ${INC_BRANCH} + echo ${REPO} + docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + displayName: 'Build Image' + + - script: | + docker images | grep -i neural-compressor + displayName: 'Check docker Image' + - script: | + docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh + displayName: 'Run UT' + diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml new file mode 100644 index 00000000000..dd7dde5c113 --- /dev/null +++ b/.azure-pipelines/ut-ncoder.yml @@ -0,0 +1,44 @@ +trigger: none + +pr: + branches: + include: + - master + path: + include: + - neural_coder + +pool: suyue-test + +variables: + PYTHON: 'python3.8' + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'pr' + +stages: +- stage: + displayName: Unit Test for Neural Coder + jobs: + - job: + variables: + INC_BRANCH: $(System.PullRequest.SourceBranch) + REPO: $(Build.Repository.Uri) + steps: + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean Docker' + - script: | + echo ${INC_BRANCH} + echo ${REPO} + docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $SYSTEM_DEFAULTWORKINGDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + displayName: 'Build Image' + + - script: | + docker images | grep -i neural-compressor + displayName: 'Check docker Image' + + - script: | + docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_ncoder.sh + displayName: 'Run UT' \ No newline at end of file diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml new file mode 100644 index 00000000000..11d4b025816 --- /dev/null +++ b/.azure-pipelines/ut-ux.yml @@ -0,0 +1,45 @@ +trigger: none + +pr: + branches: + include: + - master + path: + include: + - neural_compressor.ux + +pool: suyue-test + +variables: + PYTHON: 'python3.8' + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'pr' + +stages: +- stage: + displayName: Unit Test for UX + jobs: + - job: + variables: + INC_BRANCH: $(System.PullRequest.SourceBranch) + REPO: $(Build.Repository.Uri) + steps: + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean Docker' + + - script: | + echo ${INC_BRANCH} + echo ${REPO} + docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + displayName: 'Build Image' + + - script: | + docker images | grep -i neural-compressor + displayName: 'Check docker Image' + + - script: | + docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_ux.sh + displayName: 'Run UT' \ No newline at end of file From 2e659df87a93848a953580a27ae45c2c7d11ba4d Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Sun, 7 Aug 2022 22:12:42 +0800 Subject: [PATCH 031/296] run adaptor test --- .azure-pipelines/scripts/ut/env_setup.sh | 36 ++++++ .../scripts/ut/run_basic_adaptor.sh | 16 +++ .../ut/{run_basic.sh => run_basic_others.sh} | 3 +- .azure-pipelines/ut-basic.yml | 114 +++++++++--------- 4 files changed, 114 insertions(+), 55 deletions(-) create mode 100644 .azure-pipelines/scripts/ut/env_setup.sh rename .azure-pipelines/scripts/ut/{run_basic.sh => run_basic_others.sh} (83%) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh new file mode 100644 index 00000000000..4391db81842 --- /dev/null +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +if [[ "${tensorflow_version}" == *"-official" ]]; then + pip install tensorflow==${tensorflow_version%-official} +elif [[ "${tensorflow_version}" != "" ]]; then + pip install intel-tensorflow==${tensorflow_version} + pip install protobuf==3.20.1 +fi + +pip install torch==${pytorch_version} -f https://download.pytorch.org/whl/torch_stable.html +pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html + +pip install onnx==${onnx_version} +pip install onnxruntime==${onnxruntime_version} + +pip install mxnet==${mxnet_version} + +cd /neural-compressor/test +if [ -f "requirements.txt" ]; then + sed -i '/^neural-compressor/d' requirements.txt + sed -i '/^intel-tensorflow/d' requirements.txt + sed -i '/find-links https:\/\/download.pytorch.org\/whl\/torch_stable.html/d' requirements.txt + sed -i '/^torch/d' requirements.txt + sed -i '/^mxnet-mkl/d' requirements.txt + sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt + n=0 + until [ "$n" -ge 3 ] + do + python -m pip install --no-cache-dir -r requirements.txt && break + n=$((n+1)) + sleep 5 + done + pip list +else + echo "Not found requirements.txt file." +fi diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index b1b4ea1df24..7c46e0436e1 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -2,3 +2,19 @@ python -c "import neural_compressor as nc;print(nc.version.__version__)" echo "run basic adaptor" + +echo "set up UT env..." +bash env_setup.sh + +cd adaptor +find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +ut_log_name=${WORKSPACE}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh +echo "-------------" +bash run.sh 2>&1 | tee ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh similarity index 83% rename from .azure-pipelines/scripts/ut/run_basic.sh rename to .azure-pipelines/scripts/ut/run_basic_others.sh index ec312e3a26e..de450dd2a61 100644 --- a/.azure-pipelines/scripts/ut/run_basic.sh +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -1,4 +1,5 @@ #!/bin/bash set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" -echo "run basic" \ No newline at end of file +echo "run basic" + diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 16c3dc4109d..900f5f2f22b 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -16,6 +16,12 @@ variables: PYTHON: 'python3.8' IMAGE_NAME: 'neural-compressor' IMAGE_TAG: 'pr' + tensorflow_version: '2.9.1' + pytorch_version: '1.11.0+cpu' + torchvision_version: '0.12.0' + onnx_version: '1.9.0' + onnxruntime_version: '1.10.0' + mxnet_version: '1.7.0' stages: - stage: @@ -47,58 +53,58 @@ stages: docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh displayName: 'Run UT' -- stage: - displayName: Unit Test tf newAPI - dependsOn: [] - jobs: - - job: - variables: - INC_BRANCH: $(System.PullRequest.SourceBranch) - REPO: $(Build.Repository.Uri) - steps: - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - echo ${INC_BRANCH} - echo ${REPO} - docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - displayName: 'Build Image' - - - script: | - docker images | grep -i neural-compressor - displayName: 'Check docker Image' - - - script: | - docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh - displayName: 'Run UT' - -- stage: - displayName: Unit Test other basic case - dependsOn: [] - jobs: - - job: - variables: - INC_BRANCH: $(System.PullRequest.SourceBranch) - REPO: $(Build.Repository.Uri) - steps: - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - echo ${INC_BRANCH} - echo ${REPO} - docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - displayName: 'Build Image' - - - script: | - docker images | grep -i neural-compressor - displayName: 'Check docker Image' - - script: | - docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh - displayName: 'Run UT' +#- stage: +# displayName: Unit Test tf newAPI +# dependsOn: [] +# jobs: +# - job: +# variables: +# INC_BRANCH: $(System.PullRequest.SourceBranch) +# REPO: $(Build.Repository.Uri) +# steps: +# - script: | +# docker stop $(docker ps -aq) +# docker rm -vf $(docker ps -aq) || true +# displayName: 'Clean Docker' +# +# - script: | +# echo ${INC_BRANCH} +# echo ${REPO} +# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . +# displayName: 'Build Image' +# +# - script: | +# docker images | grep -i neural-compressor +# displayName: 'Check docker Image' +# +# - script: | +# docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh +# displayName: 'Run UT' +# +#- stage: +# displayName: Unit Test other basic case +# dependsOn: [] +# jobs: +# - job: +# variables: +# INC_BRANCH: $(System.PullRequest.SourceBranch) +# REPO: $(Build.Repository.Uri) +# steps: +# - script: | +# docker stop $(docker ps -aq) +# docker rm -vf $(docker ps -aq) || true +# displayName: 'Clean Docker' +# +# - script: | +# echo ${INC_BRANCH} +# echo ${REPO} +# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . +# displayName: 'Build Image' +# +# - script: | +# docker images | grep -i neural-compressor +# displayName: 'Check docker Image' +# - script: | +# docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_others.sh +# displayName: 'Run UT' From 190c4a4dded05b6e62186eddf8fbde7a144f7ed5 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Sun, 7 Aug 2022 22:13:30 +0800 Subject: [PATCH 032/296] for test --- neural_compressor/version.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neural_compressor/version.py b/neural_compressor/version.py index aa09939a643..d9478f11caa 100644 --- a/neural_compressor/version.py +++ b/neural_compressor/version.py @@ -14,5 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# for test __version__ = "1.13" From b9822a0c840e20317f0dc5a4a781737c02b0bb1d Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Sun, 7 Aug 2022 22:34:36 +0800 Subject: [PATCH 033/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 252d4d60056..8c045913eac 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && python setup.py install && pip list && python test/test_basic.py" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py install && pip list && python test/test_algorithm.py" displayName: 'find repo' - script: | From 5c83569e47f46625f9e7ee329a1670ebfa341212 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Sun, 7 Aug 2022 23:30:46 +0800 Subject: [PATCH 034/296] test --- .azure-pipelines/scripts/ut/run_basic_adaptor.sh | 1 + .azure-pipelines/ut-basic.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index 7c46e0436e1..ab7227784bc 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" echo "run basic adaptor" diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 900f5f2f22b..73d554f8238 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -50,7 +50,7 @@ stages: displayName: 'Check docker image' - script: | - docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh + docker run --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh displayName: 'Run UT' #- stage: From eade71931dec88c18eba812383df68468b6be3c7 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Sun, 7 Aug 2022 23:48:14 +0800 Subject: [PATCH 035/296] print env --- .azure-pipelines/scripts/ut/run_basic_adaptor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index ab7227784bc..b786d9d4457 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -3,7 +3,7 @@ set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" echo "run basic adaptor" - +env | sort echo "set up UT env..." bash env_setup.sh From 5bd154a41e8a5b00946821c403161a3e857f6262 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 00:11:51 +0800 Subject: [PATCH 036/296] update --- .azure-pipelines/scripts/ut/run_basic_adaptor.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index b786d9d4457..05f84521e57 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -3,11 +3,19 @@ set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" echo "run basic adaptor" + env | sort +echo ${tensorflow_version} +echo ${pytorch_version} +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' +export mxnet_version='1.7.0' + + echo "set up UT env..." -bash env_setup.sh +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh -cd adaptor +cd /neural-compressor/test/adaptor || exit 1 find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh ut_log_name=${WORKSPACE}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log From 6ee7533983223854d37469b3a4bc07f6c3bba9a3 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 00:38:18 +0800 Subject: [PATCH 037/296] remove image --- .azure-pipelines/ut-basic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 73d554f8238..08daabd7aa1 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -36,6 +36,7 @@ stages: - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true + docker image rm neural-compressor:pr || true env | sort displayName: 'Clean Docker' From 9deaefdcbe05d302bb450e506761030373781afe Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 11:25:00 +0800 Subject: [PATCH 038/296] test for python --- .azure-pipelines/docker/Dockerfile.devel | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index e7ef3909552..34d6f48a186 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -28,18 +28,17 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ python3 \ python3-pip \ - ${PYTHON}-dev \ - ${PYTHON}-distutils \ + python3-dev \ + python3-distutils \ autoconf \ build-essential \ git \ libgl1-mesa-glx \ libglib2.0-0 -RUN ln -sf $(which ${PYTHON}) /usr/bin/python && \ - ln -sf $(which ${PYTHON}) /usr/bin/python3 +RUN ln -sf $(which python3) /usr/bin/python -RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \ +RUN python -m pip --no-cache-dir install --upgrade \ pip \ setuptools @@ -49,5 +48,7 @@ RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compresso python -m pip install --no-cache-dir -r requirements.txt && \ python setup.py install +RUN pip list + WORKDIR /neural-compressor RUN git rev-parse HEAD From a906f4637ac1d37222083a47e2ff30ed97045b99 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 11:32:02 +0800 Subject: [PATCH 039/296] test for python --- .azure-pipelines/docker/Dockerfile.devel | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index 34d6f48a186..ca8a11eda31 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -13,13 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG UBUNTU_VER=20.04 +ARG UBUNTU_VER=22.04 FROM ubuntu:${UBUNTU_VER} as devel # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 -ARG PYTHON=python3.8 ARG REPO=x ARG TARGET_BRANCH=y ARG SOURCE_BRANCH=z @@ -48,7 +47,7 @@ RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compresso python -m pip install --no-cache-dir -r requirements.txt && \ python setup.py install -RUN pip list +RUN pip list | grep neural WORKDIR /neural-compressor RUN git rev-parse HEAD From bcb42e55cc4c4f057b9b55381f6e776dc2bbb272 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 11:38:26 +0800 Subject: [PATCH 040/296] test for python --- .azure-pipelines/docker/Dockerfile.devel | 3 ++- .azure-pipelines/ut-basic.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index ca8a11eda31..a1ab4886017 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG UBUNTU_VER=22.04 +ARG UBUNTU_VER=20.04 FROM ubuntu:${UBUNTU_VER} as devel # See http://bugs.python.org/issue19846 @@ -47,6 +47,7 @@ RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compresso python -m pip install --no-cache-dir -r requirements.txt && \ python setup.py install +RUN pip list RUN pip list | grep neural WORKDIR /neural-compressor diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 08daabd7aa1..8d7a6786527 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -43,7 +43,7 @@ stages: - script: | echo ${INC_BRANCH} echo ${REPO} - docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build --build-arg --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . displayName: 'Build Image' - script: | From 6fc988a54c0fa90a1709ea450e8517d96dd2a96e Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 11:40:59 +0800 Subject: [PATCH 041/296] bug fix --- .azure-pipelines/ut-basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 8d7a6786527..4cbd5e908ad 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -43,7 +43,7 @@ stages: - script: | echo ${INC_BRANCH} echo ${REPO} - docker build --build-arg --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . displayName: 'Build Image' - script: | From 40924ede0e918d164d70c0d9ebaf27de2e8c0b28 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 11:45:31 +0800 Subject: [PATCH 042/296] test --- .azure-pipelines/docker/Dockerfile.devel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index a1ab4886017..2c670f051ca 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -38,11 +38,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ RUN ln -sf $(which python3) /usr/bin/python RUN python -m pip --no-cache-dir install --upgrade \ - pip \ - setuptools + pip RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ cd neural-compressor && \ + python -m pip install --no-cache-dir setuptools && \ python -m pip install --no-cache-dir pycocotools && \ python -m pip install --no-cache-dir -r requirements.txt && \ python setup.py install From d24f9a10721e260e5cc42a0c814ba8b6a558f387 Mon Sep 17 00:00:00 2001 From: "chen,suyue" Date: Mon, 8 Aug 2022 11:55:07 +0800 Subject: [PATCH 043/296] test --- .azure-pipelines/docker/Dockerfile.devel | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index 2c670f051ca..cdec5db4e34 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -37,8 +37,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ RUN ln -sf $(which python3) /usr/bin/python -RUN python -m pip --no-cache-dir install --upgrade \ - pip +RUN python -m pip --no-cache-dir install --upgrade pip RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ cd neural-compressor && \ From 65c516f82badab6a446e95fce048f02c9754e89e Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 12:03:36 +0800 Subject: [PATCH 044/296] Update Dockerfile.devel --- docker/Dockerfile.devel | 46 ++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index 11204ca98db..f46e15967a5 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -12,52 +12,38 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - ARG UBUNTU_VER=20.04 FROM ubuntu:${UBUNTU_VER} as devel - # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 -ARG PYTHON=python3.8 -ARG DEBIAN_FRONTEND=noninteractive - +ARG REPO=x +ARG TARGET_BRANCH=y +ARG SOURCE_BRANCH=z RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ python3 \ - python3-pip - -RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \ - pip \ - setuptools - -RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \ - ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \ - ln -sf $(which ${PYTHON}) /usr/bin/python && \ - ln -sf $(which ${PYTHON}) /usr/bin/python3 - -RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ - ${PYTHON}-dev \ - ${PYTHON}-distutils \ + python3-pip \ + python3-dev \ + python3-distutils \ autoconf \ build-essential \ - cmake \ - g++ \ git \ libgl1-mesa-glx \ libglib2.0-0 -#ARG INC_BRANCH=v1.12 -#RUN git clone --single-branch --branch=${INC_BRANCH} https://github.com/intel/neural-compressor.git && \ +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip --no-cache-dir install --upgrade pip +RUN python -m pip install --no-cache-dir setuptools + +#RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ # cd neural-compressor && \ -# git submodule sync && \ -# git submodule update --init --recursive && \ +# python -m pip install --no-cache-dir setuptools && \ # python -m pip install --no-cache-dir pycocotools && \ # python -m pip install --no-cache-dir -r requirements.txt && \ # python setup.py install +RUN pip list +RUN pip list | grep neural + WORKDIR / -#COPY ./ /neural_compressor -#RUN pwd -#RUN ls -l -#RUN ${PYTHON} setup.py install -#WORKDIR /neural-compressor From 4d67f9564adfec277a4d01e00b546daeafe2fb6f Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 12:29:54 +0800 Subject: [PATCH 045/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 8c045913eac..bfb42dc756b 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir pycocotools && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py install && pip list && python test/test_algorithm.py" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir setuptools && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py install && pip list && python test/test_algorithm.py" displayName: 'find repo' - script: | From a7a60a97852cb3cff275565656e43ef7d712d079 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 13:23:35 +0800 Subject: [PATCH 046/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index bfb42dc756b..f6cb5a9d582 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir setuptools && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py install && pip list && python test/test_algorithm.py" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py install && pip list && python test/test_algorithm.py" displayName: 'find repo' - script: | From f68fd52749b2996bccc72b50f4a5cf12652a73bc Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 14:48:14 +0800 Subject: [PATCH 047/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index f6cb5a9d582..dac4e136921 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py install && pip list && python test/test_algorithm.py" + docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-1.13-py3-none-any.whl && pip list && python test/strategy/test_bayesian.py" displayName: 'find repo' - script: | From 4d83173e04b884ed4f3bbf1af12dcdbbc3eba61b Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 17:42:10 +0800 Subject: [PATCH 048/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index dac4e136921..c6c776420a5 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -23,7 +23,7 @@ stages: - script: | pwd ls -l - docker run -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-1.13-py3-none-any.whl && pip list && python test/strategy/test_bayesian.py" + docker run --rm -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-1.13-py3-none-any.whl && pip list && python test/strategy/test_bayesian.py" displayName: 'find repo' - script: | @@ -32,5 +32,6 @@ stages: displayName: 'Check docker' - script: | - sudo rm -fr /home/azureuser-inc/myagent/workspace/1/s/* + #sudo rm -fr /home/azureuser-inc/myagent/workspace/1/s/* + echo "clean up" displayName: 'clean up' From 49d6585d67ccc9ff511063e971e19dcaa7b65293 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:03:09 +0800 Subject: [PATCH 049/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index c6c776420a5..80ba51b5478 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -19,7 +19,10 @@ stages: jobs: - job: Build displayName: Build - steps: + steps: + - checkout: self + clean: true + - script: | pwd ls -l @@ -32,6 +35,5 @@ stages: displayName: 'Check docker' - script: | - #sudo rm -fr /home/azureuser-inc/myagent/workspace/1/s/* - echo "clean up" + sudo rm -fr /home/azureuser-inc/myagent/workspace/1/s/* displayName: 'clean up' From 6afaaa74aba6e910346d2b2b04e95b9ff389b523 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:10:13 +0800 Subject: [PATCH 050/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 80ba51b5478..b501b6eb6e9 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -20,9 +20,14 @@ stages: - job: Build displayName: Build steps: + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + - checkout: self clean: true - + - script: | pwd ls -l From 16eca55a380bd1b304a5cc9e40d223feacfcd85e Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:16:44 +0800 Subject: [PATCH 051/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index b501b6eb6e9..4b49e14e340 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -17,6 +17,10 @@ stages: - stage: Build displayName: Build and push stage jobs: + - job: Clean + workspace: + clean: all + - job: Build displayName: Build steps: @@ -24,7 +28,7 @@ stages: docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: 'Clean Docker' - + - checkout: self clean: true From 5bcb46ed69ed6dd8e0dc33738d6a8b79f1e7c692 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:23:11 +0800 Subject: [PATCH 052/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 4b49e14e340..f3ea8af14d9 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -18,8 +18,10 @@ stages: displayName: Build and push stage jobs: - job: Clean - workspace: - clean: all + steps: + - script: sudo rm -fr $(Agent.BuildDirectory) || true + #workspace: + # clean: all - job: Build displayName: Build From 660aa6fdf8c3b0a9fb9312e3cc2739ccd0ea78f4 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:25:15 +0800 Subject: [PATCH 053/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index f3ea8af14d9..1e659d8cb96 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -19,7 +19,9 @@ stages: jobs: - job: Clean steps: + - checkout: none - script: sudo rm -fr $(Agent.BuildDirectory) || true + - script: echo $(Pipeline.Workspace) && echo $(Agent.BuildDirectory) #workspace: # clean: all From 8fdacae27ef0d05e16c09b876b93682da0676621 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:28:46 +0800 Subject: [PATCH 054/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 1e659d8cb96..bca5cd0fff4 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -21,24 +21,19 @@ stages: steps: - checkout: none - script: sudo rm -fr $(Agent.BuildDirectory) || true - - script: echo $(Pipeline.Workspace) && echo $(Agent.BuildDirectory) - #workspace: - # clean: all - - - job: Build - displayName: Build - steps: + displayName: 'Clean workspace' - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: 'Clean Docker' - + + - job: Build + displayName: Build + steps: - checkout: self clean: true - script: | - pwd - ls -l docker run --rm -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-1.13-py3-none-any.whl && pip list && python test/strategy/test_bayesian.py" displayName: 'find repo' From 00f155edb00c7278b67b4fbff7e01a6e608acf74 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 20:32:34 +0800 Subject: [PATCH 055/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index bca5cd0fff4..83ca3a5a056 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -22,17 +22,17 @@ stages: - checkout: none - script: sudo rm -fr $(Agent.BuildDirectory) || true displayName: 'Clean workspace' - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - job: Build displayName: Build steps: - checkout: self clean: true - + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + - script: | docker run --rm -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-1.13-py3-none-any.whl && pip list && python test/strategy/test_bayesian.py" displayName: 'find repo' From 0bb62c734ba1c43e7a7ceda476acd684deebfc2f Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 22:37:54 +0800 Subject: [PATCH 056/296] Update Dockerfile.devel --- docker/Dockerfile.devel | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index f46e15967a5..b692fea48e5 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -17,9 +17,7 @@ FROM ubuntu:${UBUNTU_VER} as devel # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 -ARG REPO=x -ARG TARGET_BRANCH=y -ARG SOURCE_BRANCH=z + RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ python3 \ python3-pip \ @@ -44,6 +42,6 @@ RUN python -m pip install --no-cache-dir setuptools # python setup.py install RUN pip list -RUN pip list | grep neural +# RUN pip list | grep neural WORKDIR / From 2599a350c300001285cd01f0603dcc7a3c136f5b Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:19:24 +0800 Subject: [PATCH 057/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 83ca3a5a056..7be48da4b34 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -17,31 +17,35 @@ stages: - stage: Build displayName: Build and push stage jobs: - - job: Clean - steps: - - checkout: none - - script: sudo rm -fr $(Agent.BuildDirectory) || true - displayName: 'Clean workspace' - - job: Build displayName: Build steps: + - checkout: none + + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + - checkout: self - clean: true + + - script: | + if [[ ! $(docker images | grep -i neural-compressor) ]]; then + docker build -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + displayName: "build images" + - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: 'Clean Docker' - + - script: | - docker run --rm -v /home/azureuser-inc/myagent/workspace/1/s:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-1.13-py3-none-any.whl && pip list && python test/strategy/test_bayesian.py" + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && python test/strategy/test_bayesian.py" displayName: 'find repo' - script: | docker images | grep -i neural-compressor docker ps displayName: 'Check docker' - - - script: | - sudo rm -fr /home/azureuser-inc/myagent/workspace/1/s/* - displayName: 'clean up' + From 4d981d3c884c3d3b84231005666315e9751d7a4a Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:27:14 +0800 Subject: [PATCH 058/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 7be48da4b34..902e39d1b20 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -27,7 +27,9 @@ stages: sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true displayName: 'Clean workspace' - - checkout: self + - script: | + git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin + displayName: "checkout repo" - script: | if [[ ! $(docker images | grep -i neural-compressor) ]]; then From 9f4c5a2ae3e1b28729fbc71781654bd3383903ff Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:31:47 +0800 Subject: [PATCH 059/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 902e39d1b20..ac5f487eb64 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -28,7 +28,7 @@ stages: displayName: 'Clean workspace' - script: | - git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin + git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} displayName: "checkout repo" - script: | From 40336db2707a3f305d2c3aeb92582a567c15cbd8 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:37:18 +0800 Subject: [PATCH 060/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index ac5f487eb64..79389431355 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -28,7 +28,9 @@ stages: displayName: 'Clean workspace' - script: | - git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git clone --single-branch --branch=$(System.PullRequest.SourceBranch) $(Build.Repository.Uri) + #git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} displayName: "checkout repo" - script: | From c663af84f156dbabf958cdaee4f5826f836c2b9e Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:39:49 +0800 Subject: [PATCH 061/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 79389431355..7cfbca35191 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -28,7 +28,7 @@ stages: displayName: 'Clean workspace' - script: | - cd ${BUILD_SOURCESDIRECTORY} + mkdir -p ${BUILD_SOURCESDIRECTORY} && cd ${BUILD_SOURCESDIRECTORY} git clone --single-branch --branch=$(System.PullRequest.SourceBranch) $(Build.Repository.Uri) #git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} displayName: "checkout repo" From a7b771755f94ac73782bff26025267dbbea408ab Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:41:18 +0800 Subject: [PATCH 062/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 7cfbca35191..97af95bb87a 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -28,7 +28,7 @@ stages: displayName: 'Clean workspace' - script: | - mkdir -p ${BUILD_SOURCESDIRECTORY} && cd ${BUILD_SOURCESDIRECTORY} + #mkdir -p ${BUILD_SOURCESDIRECTORY} && cd ${BUILD_SOURCESDIRECTORY} git clone --single-branch --branch=$(System.PullRequest.SourceBranch) $(Build.Repository.Uri) #git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} displayName: "checkout repo" From fae699f938fa9500c22e391952d8c20677105f50 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:43:53 +0800 Subject: [PATCH 063/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 97af95bb87a..36783b7f762 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -20,16 +20,18 @@ stages: - job: Build displayName: Build steps: - - checkout: none - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true displayName: 'Clean workspace' + - checkout: self + - script: | + echo "checkout repo" #mkdir -p ${BUILD_SOURCESDIRECTORY} && cd ${BUILD_SOURCESDIRECTORY} - git clone --single-branch --branch=$(System.PullRequest.SourceBranch) $(Build.Repository.Uri) + #git clone --single-branch --branch=$(System.PullRequest.SourceBranch) $(Build.Repository.Uri) #git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} displayName: "checkout repo" From a999df52499d5f8c99b2deee75a7bc53077dfc72 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Mon, 8 Aug 2022 23:45:49 +0800 Subject: [PATCH 064/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 36783b7f762..5dbfa8e2542 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -27,7 +27,7 @@ stages: displayName: 'Clean workspace' - checkout: self - + - script: | echo "checkout repo" #mkdir -p ${BUILD_SOURCESDIRECTORY} && cd ${BUILD_SOURCESDIRECTORY} @@ -36,6 +36,7 @@ stages: displayName: "checkout repo" - script: | + sudo chmod 666 /var/run/docker.sock if [[ ! $(docker images | grep -i neural-compressor) ]]; then docker build -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi From add65e47a254cb57607875158190faa828b3d7fb Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 11:06:11 +0800 Subject: [PATCH 065/296] Update azure-pipelines-1.yml for Azure Pipelines --- azure-pipelines-1.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml index 5dbfa8e2542..843c6c1d9e7 100644 --- a/azure-pipelines-1.yml +++ b/azure-pipelines-1.yml @@ -27,20 +27,13 @@ stages: displayName: 'Clean workspace' - checkout: self - - - script: | - echo "checkout repo" - #mkdir -p ${BUILD_SOURCESDIRECTORY} && cd ${BUILD_SOURCESDIRECTORY} - #git clone --single-branch --branch=$(System.PullRequest.SourceBranch) $(Build.Repository.Uri) - #git -c fetch --force --tags --prune --prune-tags --progress --no-recurse-submodules origin ${BUILD_SOURCESDIRECTORY} - displayName: "checkout repo" + displayName: "Checkout out Repo" - script: | - sudo chmod 666 /var/run/docker.sock if [[ ! $(docker images | grep -i neural-compressor) ]]; then docker build -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi - displayName: "build images" + displayName: "Build Devel Images" - script: | docker stop $(docker ps -aq) @@ -49,7 +42,7 @@ stages: - script: | docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && python test/strategy/test_bayesian.py" - displayName: 'find repo' + displayName: 'Build Binary and Run UT' - script: | docker images | grep -i neural-compressor From 00c5a1efa2e5107aecead97b76af0933a1d19b2d Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 11:26:44 +0800 Subject: [PATCH 066/296] Rename azure-pipelines-1.yml to azure-pipelines-wenxin.yml --- azure-pipelines-1.yml => azure-pipelines-wenxin.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename azure-pipelines-1.yml => azure-pipelines-wenxin.yml (100%) diff --git a/azure-pipelines-1.yml b/azure-pipelines-wenxin.yml similarity index 100% rename from azure-pipelines-1.yml rename to azure-pipelines-wenxin.yml From d0ecc8ab93570a1e5534962e087cb51ba1719f2d Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 11:50:23 +0800 Subject: [PATCH 067/296] Update azure-pipelines-wenxin.yml for Azure Pipelines --- azure-pipelines-wenxin.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines-wenxin.yml b/azure-pipelines-wenxin.yml index 843c6c1d9e7..5a55b6b0d3b 100644 --- a/azure-pipelines-wenxin.yml +++ b/azure-pipelines-wenxin.yml @@ -9,7 +9,6 @@ pr: pool: docker-agent variables: - PYTHON: 'python3.8' IMAGE_NAME: 'neural-compressor' IMAGE_TAG: 'py38' @@ -41,7 +40,7 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && pip install intel-tensorflow && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && python test/strategy/test_bayesian.py" + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" displayName: 'Build Binary and Run UT' - script: | From c467c3b450a083fb750274a53bbce65719b517e3 Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 14:12:27 +0800 Subject: [PATCH 068/296] Update and rename azure-pipelines-wenxin.yml to .azure-pipelines/azure-pipelines-demo.yml --- .../azure-pipelines-demo.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) rename azure-pipelines-wenxin.yml => .azure-pipelines/azure-pipelines-demo.yml (95%) diff --git a/azure-pipelines-wenxin.yml b/.azure-pipelines/azure-pipelines-demo.yml similarity index 95% rename from azure-pipelines-wenxin.yml rename to .azure-pipelines/azure-pipelines-demo.yml index 5a55b6b0d3b..7ad0938dd32 100644 --- a/azure-pipelines-wenxin.yml +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -3,6 +3,7 @@ # Add steps that build, run tests, deploy, and more: # https://aka.ms/yaml +trigger: none pr: - wenxin-test @@ -19,7 +20,6 @@ stages: - job: Build displayName: Build steps: - - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true @@ -32,6 +32,8 @@ stages: if [[ ! $(docker images | grep -i neural-compressor) ]]; then docker build -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi + docker images | grep -i neural-compressor + [[ $? -ne 0 ]] && exit 1 displayName: "Build Devel Images" - script: | @@ -43,8 +45,4 @@ stages: docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" displayName: 'Build Binary and Run UT' - - script: | - docker images | grep -i neural-compressor - docker ps - displayName: 'Check docker' From e739875dabe086f971a869a47bc6e40569fb3cde Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 14:14:28 +0800 Subject: [PATCH 069/296] Update Dockerfile.devel --- docker/Dockerfile.devel | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index b692fea48e5..441c68aa705 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -14,10 +14,10 @@ # limitations under the License. ARG UBUNTU_VER=20.04 FROM ubuntu:${UBUNTU_VER} as devel + # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 - RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ python3 \ python3-pip \ @@ -34,14 +34,6 @@ RUN ln -sf $(which python3) /usr/bin/python RUN python -m pip --no-cache-dir install --upgrade pip RUN python -m pip install --no-cache-dir setuptools -#RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ -# cd neural-compressor && \ -# python -m pip install --no-cache-dir setuptools && \ -# python -m pip install --no-cache-dir pycocotools && \ -# python -m pip install --no-cache-dir -r requirements.txt && \ -# python setup.py install - RUN pip list -# RUN pip list | grep neural WORKDIR / From 92111efd7a8288daf0a92d0aaa4cdd94bf555322 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 14:21:07 +0800 Subject: [PATCH 070/296] update dockerfile name --- .../docker/{Dockerfile.devel => Dockerfile-inc.devel} | 0 .azure-pipelines/ut-basic.yml | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename .azure-pipelines/docker/{Dockerfile.devel => Dockerfile-inc.devel} (100%) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile-inc.devel similarity index 100% rename from .azure-pipelines/docker/Dockerfile.devel rename to .azure-pipelines/docker/Dockerfile-inc.devel diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 4cbd5e908ad..3949fd789ef 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -43,7 +43,7 @@ stages: - script: | echo ${INC_BRANCH} echo ${REPO} - docker build --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . displayName: 'Build Image' - script: | @@ -71,7 +71,7 @@ stages: # - script: | # echo ${INC_BRANCH} # echo ${REPO} -# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . +# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . # displayName: 'Build Image' # # - script: | @@ -99,7 +99,7 @@ stages: # - script: | # echo ${INC_BRANCH} # echo ${REPO} -# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . +# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . # displayName: 'Build Image' # # - script: | From 8fcd6492a53ec0162d42e699ac82507a046537c7 Mon Sep 17 00:00:00 2001 From: sys-lpot-val Date: Wed, 10 Aug 2022 13:57:40 +0800 Subject: [PATCH 071/296] clean up --- .azure-pipelines/docker/Dockerfile.devel | 28 ++++++------------------ 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index e7ef3909552..441c68aa705 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -12,42 +12,28 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - ARG UBUNTU_VER=20.04 FROM ubuntu:${UBUNTU_VER} as devel # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 -ARG PYTHON=python3.8 -ARG REPO=x -ARG TARGET_BRANCH=y -ARG SOURCE_BRANCH=z -ARG DEBIAN_FRONTEND=noninteractive - RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ python3 \ python3-pip \ - ${PYTHON}-dev \ - ${PYTHON}-distutils \ + python3-dev \ + python3-distutils \ autoconf \ build-essential \ git \ libgl1-mesa-glx \ libglib2.0-0 -RUN ln -sf $(which ${PYTHON}) /usr/bin/python && \ - ln -sf $(which ${PYTHON}) /usr/bin/python3 +RUN ln -sf $(which python3) /usr/bin/python -RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \ - pip \ - setuptools +RUN python -m pip --no-cache-dir install --upgrade pip +RUN python -m pip install --no-cache-dir setuptools -RUN git clone --single-branch --branch=${SOURCE_BRANCH} ${REPO} neural-compressor && \ - cd neural-compressor && \ - python -m pip install --no-cache-dir pycocotools && \ - python -m pip install --no-cache-dir -r requirements.txt && \ - python setup.py install +RUN pip list -WORKDIR /neural-compressor -RUN git rev-parse HEAD +WORKDIR / From df6b7b1c9caacc890022052934f6a84a243452dc Mon Sep 17 00:00:00 2001 From: sys-lpot-val Date: Wed, 10 Aug 2022 13:59:26 +0800 Subject: [PATCH 072/296] clean up --- docker/Dockerfile.devel | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index 441c68aa705..31afb67c918 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -12,28 +12,51 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + ARG UBUNTU_VER=20.04 FROM ubuntu:${UBUNTU_VER} as devel # See http://bugs.python.org/issue19846 ENV LANG C.UTF-8 +ARG PYTHON=python3.8 +ARG DEBIAN_FRONTEND=noninteractive + RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ python3 \ - python3-pip \ - python3-dev \ - python3-distutils \ + python3-pip + +RUN ${PYTHON} -m pip --no-cache-dir install --upgrade \ + pip \ + setuptools + +RUN ln -sf $(which ${PYTHON}) /usr/local/bin/python && \ + ln -sf $(which ${PYTHON}) /usr/local/bin/python3 && \ + ln -sf $(which ${PYTHON}) /usr/bin/python && \ + ln -sf $(which ${PYTHON}) /usr/bin/python3 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + ${PYTHON}-dev \ + ${PYTHON}-distutils \ autoconf \ build-essential \ + cmake \ + g++ \ git \ libgl1-mesa-glx \ libglib2.0-0 -RUN ln -sf $(which python3) /usr/bin/python - -RUN python -m pip --no-cache-dir install --upgrade pip -RUN python -m pip install --no-cache-dir setuptools +ARG INC_BRANCH=v1.12 +RUN git clone --single-branch --branch=${INC_BRANCH} https://github.com/intel/neural-compressor.git && \ + cd neural-compressor && \ + git submodule sync && \ + git submodule update --init --recursive && \ + python -m pip install --no-cache-dir pycocotools && \ + python -m pip install --no-cache-dir -r requirements.txt && \ + python setup.py install -RUN pip list +WORKDIR /neural-compressor -WORKDIR / +ADD https://raw.githubusercontent.com/intel/neural-compressor/v${INC_VER}/third-party-programs.txt /licenses/ +ADD https://raw.githubusercontent.com/intel/neural-compressor/v${INC_VER}/docker/third-party-programs-docker.txt /licenses/ +ADD https://raw.githubusercontent.com/intel/neural-compressor/v${INC_VER}/LICENSE /licenses/ From de4a174bbd80cad7d32e074bf861b675788e02d1 Mon Sep 17 00:00:00 2001 From: sys-lpot-val Date: Wed, 10 Aug 2022 14:01:51 +0800 Subject: [PATCH 073/296] clean up --- .azure-pipelines/azure-pipelines-demo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml index 7ad0938dd32..b9fd5baf592 100644 --- a/.azure-pipelines/azure-pipelines-demo.yml +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -30,7 +30,7 @@ stages: - script: | if [[ ! $(docker images | grep -i neural-compressor) ]]; then - docker build -f docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i neural-compressor [[ $? -ne 0 ]] && exit 1 From 2236cba8ad1a837ea2a63856e7fda47848485bcf Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 14:44:00 +0800 Subject: [PATCH 074/296] Update azure-pipelines-demo.yml --- .azure-pipelines/azure-pipelines-demo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml index b9fd5baf592..7c1ae3953fe 100644 --- a/.azure-pipelines/azure-pipelines-demo.yml +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -42,7 +42,7 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && python test/strategy/test_bayesian.py" displayName: 'Build Binary and Run UT' From ce20191cd47802c348ff71db0004ea3c707cc26f Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 14:48:25 +0800 Subject: [PATCH 075/296] Update azure-pipelines-demo.yml for Azure Pipelines --- .azure-pipelines/azure-pipelines-demo.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml index 7c1ae3953fe..0fc6a1968f4 100644 --- a/.azure-pipelines/azure-pipelines-demo.yml +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -29,11 +29,14 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i neural-compressor) ]]; then + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi - docker images | grep -i neural-compressor - [[ $? -ne 0 ]] && exit 1 + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi displayName: "Build Devel Images" - script: | From f0f27c95ab7e6a841889b1313b123f39d8df06fa Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 16:07:33 +0800 Subject: [PATCH 076/296] Update azure-pipelines-demo.yml --- .azure-pipelines/azure-pipelines-demo.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml index 0fc6a1968f4..b6eb669b2f6 100644 --- a/.azure-pipelines/azure-pipelines-demo.yml +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -45,7 +45,7 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && python test/strategy/test_bayesian.py" + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" displayName: 'Build Binary and Run UT' From 123e210681b01b6213b937468772565f74f4ac6c Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 9 Aug 2022 17:06:27 +0800 Subject: [PATCH 077/296] Update README.md --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index dd8b285c1ab..dc3923e1ec4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ https://pypi.org/project/neural-compressor/#history ```console $ PYTHON=python3.8 -$ INC_VER=1.12 +$ INC_VER=1.13 $ IMAGE_NAME=neural-compressor $ IMAGE_TAG=${INC_VER} $ docker build --build-arg PYTHON=${PYTHON} --build-arg INC_VER=${INC_VER} -f Dockerfile -t ${IMAGE_NAME}:${IMAGE_TAG} . @@ -20,7 +20,7 @@ then please modify the tag so that the tagging requirement is met. For example r ```console $ PYTHON=python3.8 -$ INC_BRANCH=v1.12 +$ INC_BRANCH=v1.13 $ IMAGE_NAME=neural-compressor $ IMAGE_TAG=${INC_BRANCH}-devel $ docker build --build-arg PYTHON=${PYTHON} --build-arg INC_BRANCH=${INC_BRANCH} -f Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . From 13ed112d37b03bfad2c22e58cf91b0ef417ce30e Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 21:58:47 +0800 Subject: [PATCH 078/296] resolve conflict --- .../docker/{Dockerfile-inc.devel => DockerfileWithNC.devel} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .azure-pipelines/docker/{Dockerfile-inc.devel => DockerfileWithNC.devel} (98%) diff --git a/.azure-pipelines/docker/Dockerfile-inc.devel b/.azure-pipelines/docker/DockerfileWithNC.devel similarity index 98% rename from .azure-pipelines/docker/Dockerfile-inc.devel rename to .azure-pipelines/docker/DockerfileWithNC.devel index cdec5db4e34..113fdead9b9 100644 --- a/.azure-pipelines/docker/Dockerfile-inc.devel +++ b/.azure-pipelines/docker/DockerfileWithNC.devel @@ -50,4 +50,4 @@ RUN pip list RUN pip list | grep neural WORKDIR /neural-compressor -RUN git rev-parse HEAD +RUN git rev-parse HEAD \ No newline at end of file From 368cd034d9cee7f80c46baadb127b38f366e37bf Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 22:01:07 +0800 Subject: [PATCH 079/296] resolve conflict --- .azure-pipelines/docker/Dockerfile.devel | 39 ++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .azure-pipelines/docker/Dockerfile.devel diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel new file mode 100644 index 00000000000..ea08201046b --- /dev/null +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -0,0 +1,39 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +ARG UBUNTU_VER=20.04 +FROM ubuntu:${UBUNTU_VER} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + python3 \ + python3-pip \ + python3-dev \ + python3-distutils \ + autoconf \ + build-essential \ + git \ + libgl1-mesa-glx \ + libglib2.0-0 + +RUN ln -sf $(which python3) /usr/bin/python + +RUN python -m pip --no-cache-dir install --upgrade pip +RUN python -m pip install --no-cache-dir setuptools + +RUN pip list + +WORKDIR / \ No newline at end of file From 054ef05e7ed60667208e560da64f9efcc56e517d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 22:55:34 +0800 Subject: [PATCH 080/296] update ut test --- .azure-pipelines/scripts/install_nc.sh | 7 +++ .azure-pipelines/scripts/install_nc_full.sh | 7 +++ .azure-pipelines/scripts/ut/env_setup.sh | 1 - .../scripts/ut/run_basic_adaptor.sh | 15 +++---- .azure-pipelines/ut-basic.yml | 44 +++++++++---------- 5 files changed, 40 insertions(+), 34 deletions(-) create mode 100644 .azure-pipelines/scripts/install_nc.sh create mode 100644 .azure-pipelines/scripts/install_nc_full.sh diff --git a/.azure-pipelines/scripts/install_nc.sh b/.azure-pipelines/scripts/install_nc.sh new file mode 100644 index 00000000000..8fdd193d8c6 --- /dev/null +++ b/.azure-pipelines/scripts/install_nc.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /neural_compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py sdist bdist_wheel +pip install dist/neural_compressor-*.whl +pip list \ No newline at end of file diff --git a/.azure-pipelines/scripts/install_nc_full.sh b/.azure-pipelines/scripts/install_nc_full.sh new file mode 100644 index 00000000000..0e0cca549fc --- /dev/null +++ b/.azure-pipelines/scripts/install_nc_full.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /neural_compressor +python -m pip install --no-cache-dir -r requirements.txt +python setup.py --full sdist bdist_wheel +pip install dist/neural_compressor-*.whl +pip list \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 4391db81842..4b2145a7e48 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -4,7 +4,6 @@ if [[ "${tensorflow_version}" == *"-official" ]]; then pip install tensorflow==${tensorflow_version%-official} elif [[ "${tensorflow_version}" != "" ]]; then pip install intel-tensorflow==${tensorflow_version} - pip install protobuf==3.20.1 fi pip install torch==${pytorch_version} -f https://download.pytorch.org/whl/torch_stable.html diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index 05f84521e57..d3b1f592022 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -1,22 +1,19 @@ #!/bin/bash set -x -python -c "import neural_compressor as nc;print(nc.version.__version__)" -echo "run basic adaptor" - -env | sort -echo ${tensorflow_version} -echo ${pytorch_version} +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' export onnx_version='1.9.0' export onnxruntime_version='1.10.0' export mxnet_version='1.7.0' - echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh -cd /neural-compressor/test/adaptor || exit 1 -find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +cd /neural-compressor/test || exit 1 +find ./adaptor -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh ut_log_name=${WORKSPACE}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log echo "cat run.sh..." diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 3949fd789ef..ebb92afeeb6 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -13,15 +13,8 @@ pr: pool: suyue-test variables: - PYTHON: 'python3.8' IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'pr' - tensorflow_version: '2.9.1' - pytorch_version: '1.11.0+cpu' - torchvision_version: '0.12.0' - onnx_version: '1.9.0' - onnxruntime_version: '1.10.0' - mxnet_version: '1.7.0' + IMAGE_TAG: 'py38' stages: - stage: @@ -29,30 +22,33 @@ stages: dependsOn: [] jobs: - job: - variables: - INC_BRANCH: $(System.PullRequest.SourceBranch) - REPO: $(Build.Repository.Uri) steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Docker Image" - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true - docker image rm neural-compressor:pr || true env | sort displayName: 'Clean Docker' - script: | - echo ${INC_BRANCH} - echo ${REPO} - docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - displayName: 'Build Image' - - - script: | - docker images | grep -i neural-compressor - displayName: 'Check docker image' - - - script: | - docker run --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor.sh - displayName: 'Run UT' + 'docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} + bash -x -c "cd /neural-compressor/.azure-pipelines/scripts + && bash install_nc.sh + && bash ut/run_basic_adaptor.sh"' + displayName: 'Install Neural Compressor and Run UT' #- stage: # displayName: Unit Test tf newAPI From 94ae9f21be43958dd28a0e2054582c80560a18ee Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:21:12 +0800 Subject: [PATCH 081/296] for test --- .azure-pipelines/ut-basic.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index ebb92afeeb6..0ecb3fbc56a 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -23,10 +23,6 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - script: | if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . @@ -65,8 +61,6 @@ stages: # displayName: 'Clean Docker' # # - script: | -# echo ${INC_BRANCH} -# echo ${REPO} # docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . # displayName: 'Build Image' # From 085576c19de071a33655ee0b0be6ce2562daba6d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:23:21 +0800 Subject: [PATCH 082/296] fix bug --- .azure-pipelines/ut-basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 0ecb3fbc56a..9f822a33545 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -24,7 +24,7 @@ stages: - job: steps: - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} From 7be4c4c449e781342cbd56f8d8ed7dce3d0bc3be Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:31:20 +0800 Subject: [PATCH 083/296] bug fix --- .azure-pipelines/ut-basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 9f822a33545..36d69361031 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -40,7 +40,7 @@ stages: displayName: 'Clean Docker' - script: | - 'docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} + 'docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash -x -c "cd /neural-compressor/.azure-pipelines/scripts && bash install_nc.sh && bash ut/run_basic_adaptor.sh"' From 17fded4760e44d81af740495d378f683b9aec3ed Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:34:40 +0800 Subject: [PATCH 084/296] test --- .azure-pipelines/ut-basic.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 36d69361031..cb373323f9c 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -40,10 +40,10 @@ stages: displayName: 'Clean Docker' - script: | - 'docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} - bash -x -c "cd /neural-compressor/.azure-pipelines/scripts - && bash install_nc.sh - && bash ut/run_basic_adaptor.sh"' + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} + bash -x -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor.sh" displayName: 'Install Neural Compressor and Run UT' #- stage: From 9482fe7af24daa8a5c7d69f235132ca5b5754529 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:40:59 +0800 Subject: [PATCH 085/296] test 1 --- .azure-pipelines/ut-basic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index cb373323f9c..11f1b813438 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -40,8 +40,8 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} - bash -x -c "cd /neural-compressor/.azure-pipelines/scripts \ + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc.sh \ && bash ut/run_basic_adaptor.sh" displayName: 'Install Neural Compressor and Run UT' From 9827f74a5c3aedbc6ddc80d6ce4cc20b59d2f860 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:47:15 +0800 Subject: [PATCH 086/296] bug fix --- .azure-pipelines/scripts/install_nc.sh | 2 +- .azure-pipelines/scripts/install_nc_full.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/install_nc.sh b/.azure-pipelines/scripts/install_nc.sh index 8fdd193d8c6..5b6fdc6460d 100644 --- a/.azure-pipelines/scripts/install_nc.sh +++ b/.azure-pipelines/scripts/install_nc.sh @@ -1,6 +1,6 @@ #!/bin/bash -cd /neural_compressor +cd /neural-compressor python -m pip install --no-cache-dir -r requirements.txt python setup.py sdist bdist_wheel pip install dist/neural_compressor-*.whl diff --git a/.azure-pipelines/scripts/install_nc_full.sh b/.azure-pipelines/scripts/install_nc_full.sh index 0e0cca549fc..b99d2ab305f 100644 --- a/.azure-pipelines/scripts/install_nc_full.sh +++ b/.azure-pipelines/scripts/install_nc_full.sh @@ -1,6 +1,6 @@ #!/bin/bash -cd /neural_compressor +cd /neural-compressor python -m pip install --no-cache-dir -r requirements.txt python setup.py --full sdist bdist_wheel pip install dist/neural_compressor-*.whl From 1cc8d7c81f8cbb5c99154aa51978cccbdfbf02a5 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 9 Aug 2022 23:52:32 +0800 Subject: [PATCH 087/296] add log_dir --- .azure-pipelines/scripts/ut/run_basic_adaptor.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index d3b1f592022..faf89569b15 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -14,7 +14,9 @@ bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh cd /neural-compressor/test || exit 1 find ./adaptor -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh -ut_log_name=${WORKSPACE}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +LOG_DIR=/neural-compressor/log_dir +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log echo "cat run.sh..." cat run.sh From 2eacf82627cc38b7ffcd0463c983ded85119c408 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 00:14:36 +0800 Subject: [PATCH 088/296] for test --- .azure-pipelines/docker/Dockerfile.devel | 5 +- .azure-pipelines/ut-basic.yml | 110 ++++++++++++----------- 2 files changed, 62 insertions(+), 53 deletions(-) diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index e670744fa12..5e9db3f57b7 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -27,7 +27,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ build-essential \ git \ libgl1-mesa-glx \ - libglib2.0-0 + libglib2.0-0 \ + numactl \ + time \ + wget RUN ln -sf $(which python3) /usr/bin/python diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 11f1b813438..36936fa4bb1 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -24,6 +24,8 @@ stages: - job: steps: - script: | + # for test + docker image rm ${IMAGE_NAME}:${IMAGE_TAG} if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi @@ -46,56 +48,60 @@ stages: && bash ut/run_basic_adaptor.sh" displayName: 'Install Neural Compressor and Run UT' -#- stage: -# displayName: Unit Test tf newAPI -# dependsOn: [] -# jobs: -# - job: -# variables: -# INC_BRANCH: $(System.PullRequest.SourceBranch) -# REPO: $(Build.Repository.Uri) -# steps: -# - script: | -# docker stop $(docker ps -aq) -# docker rm -vf $(docker ps -aq) || true -# displayName: 'Clean Docker' -# -# - script: | -# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . -# displayName: 'Build Image' -# -# - script: | -# docker images | grep -i neural-compressor -# displayName: 'Check docker Image' -# -# - script: | -# docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh -# displayName: 'Run UT' -# -#- stage: -# displayName: Unit Test other basic case -# dependsOn: [] -# jobs: -# - job: -# variables: -# INC_BRANCH: $(System.PullRequest.SourceBranch) -# REPO: $(Build.Repository.Uri) -# steps: -# - script: | -# docker stop $(docker ps -aq) -# docker rm -vf $(docker ps -aq) || true -# displayName: 'Clean Docker' -# -# - script: | -# echo ${INC_BRANCH} -# echo ${REPO} -# docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile-inc.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . -# displayName: 'Build Image' -# -# - script: | -# docker images | grep -i neural-compressor -# displayName: 'Check docker Image' -# - script: | -# docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} /usr/bin/python /neural-compressor/.azure-pipelines/scripts/ut/run_basic_others.sh -# displayName: 'Run UT' +- stage: + displayName: Unit Test tf newAPI + dependsOn: [] + jobs: + - job: + steps: + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Docker Image" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor_tfnewapi.sh" + displayName: 'Install Neural Compressor and Run UT' + +- stage: + displayName: Unit Test other basic case + dependsOn: [] + jobs: + - job: + steps: + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Docker Image" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean Docker' + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_others.sh" + displayName: 'Install Neural Compressor and Run UT' From a98444dae136bbec1e93b98019884b004fd0d4d7 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 00:23:02 +0800 Subject: [PATCH 089/296] fix bug --- .azure-pipelines/ut-basic.yml | 122 ++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 49 deletions(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 36936fa4bb1..08328399a1a 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -23,6 +23,14 @@ stages: jobs: - job: steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + - script: | # for test docker image rm ${IMAGE_NAME}:${IMAGE_TAG} @@ -34,74 +42,90 @@ stages: echo "NO Such Image ${IMAGE_NAME}" exit 1 fi - displayName: "Build Devel Docker Image" + displayName: "Build develop docker image" - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true env | sort - displayName: 'Clean Docker' + displayName: 'Clean docker' - script: | docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc.sh \ && bash ut/run_basic_adaptor.sh" - displayName: 'Install Neural Compressor and Run UT' + displayName: 'Install Neural Compressor and run UT' - stage: displayName: Unit Test tf newAPI dependsOn: [] jobs: - - job: - steps: - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Docker Image" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean Docker' + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_adaptor_tfnewapi.sh" - displayName: 'Install Neural Compressor and Run UT' + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor_tfnewapi.sh" + displayName: 'Install Neural Compressor and run UT' - stage: displayName: Unit Test other basic case dependsOn: [] jobs: - - job: - steps: - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Docker Image" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean Docker' + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_others.sh" - displayName: 'Install Neural Compressor and Run UT' + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_others.sh" + displayName: 'Install Neural Compressor and run UT' From f103ee7f32d14473c18f4174c940c4c8c28a8486 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 00:24:27 +0800 Subject: [PATCH 090/296] fix bug --- .azure-pipelines/ut-basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 08328399a1a..d4079e0898a 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -98,7 +98,7 @@ stages: dependsOn: [] jobs: - job: - steps: + steps: - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true From 0402808c2b2f113e0e0b1443508e151874cc2378 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 00:47:35 +0800 Subject: [PATCH 091/296] update basic ut --- .azure-pipelines/scripts/ut/env_setup.sh | 24 +++++++++++---- .../scripts/ut/run_basic_adaptor.sh | 4 +-- .../scripts/ut/run_basic_adaptor_tfnewapi.sh | 23 ++++++++++++++- .../scripts/ut/run_basic_others.sh | 29 +++++++++++++++++++ .azure-pipelines/ut-basic.yml | 2 +- 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 4b2145a7e48..9be21c9024b 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -2,17 +2,31 @@ if [[ "${tensorflow_version}" == *"-official" ]]; then pip install tensorflow==${tensorflow_version%-official} +elif [[ "${tensorflow_version}" == "spr-base" ]]; then + pip install /tf_dataset/tf_binary/tensorflow*.whl elif [[ "${tensorflow_version}" != "" ]]; then pip install intel-tensorflow==${tensorflow_version} fi -pip install torch==${pytorch_version} -f https://download.pytorch.org/whl/torch_stable.html -pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html +if [[ "${pytorch_version}" != "" ]]; then + pip install torch==${pytorch_version} -f https://download.pytorch.org/whl/torch_stable.html +fi + +if [[ "${torchvision_version}" != "" ]]; then + pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html +fi -pip install onnx==${onnx_version} -pip install onnxruntime==${onnxruntime_version} +if [[ "${onnx}" != "" ]]; then + pip install onnx==${onnx_version} +fi -pip install mxnet==${mxnet_version} +if [[ "${onnxruntime}" != "" ]]; then + pip install onnxruntime==${onnxruntime_version} +fi + +if [[ "${mxnet}" != "" ]]; then + pip install mxnet==${mxnet_version} +fi cd /neural-compressor/test if [ -f "requirements.txt" ]; then diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index faf89569b15..f54f2df1117 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -19,9 +19,9 @@ LOG_DIR=/neural-compressor/log_dir ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log echo "cat run.sh..." -cat run.sh +cat run.sh | tee ${ut_log_name} echo "-------------" -bash run.sh 2>&1 | tee ${ut_log_name} +bash run.sh 2>&1 | tee -a ${ut_log_name} if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then exit 1 diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh index fdcf2730401..f0aa8e72ba5 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -1,4 +1,25 @@ #!/bin/bash set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" -echo "run basic adaptor tfnewapi" \ No newline at end of file +echo "run basic adaptor tfnewapi" + +echo "specify fwk version..." +export tensorflow_version='spr-base' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./tfnewapi -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +ut_log_name=${LOG_DIR}/ut_tf_newapi.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh index de450dd2a61..e0e98116548 100644 --- a/.azure-pipelines/scripts/ut/run_basic_others.sh +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -3,3 +3,32 @@ set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" echo "run basic" +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' +export mxnet_version='1.7.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +sed -i '/ adaptor\//d' run.sh +sed -i '/ tfnewapi\//d' run.sh +sed -i '/ ux\//d' run.sh +sed -i '/ neural_coder\//d' run.sh + +LOG_DIR=/neural-compressor/log_dir +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index d4079e0898a..e7561454836 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -87,7 +87,7 @@ stages: displayName: 'Clean docker' - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc.sh \ && bash ut/run_basic_adaptor_tfnewapi.sh" From 088bdf638e3e3f04196f7796c417525e54a3f725 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:00:32 +0800 Subject: [PATCH 092/296] add all ut test --- .azure-pipelines/scripts/ut/run_ncoder.sh | 20 +++++++++- .azure-pipelines/scripts/ut/run_ux.sh | 27 +++++++++++++- .azure-pipelines/ut-basic.yml | 3 ++ .azure-pipelines/ut-ncoder.yml | 45 ++++++++++++++--------- .azure-pipelines/ut-ux.yml | 44 +++++++++++++--------- 5 files changed, 103 insertions(+), 36 deletions(-) diff --git a/.azure-pipelines/scripts/ut/run_ncoder.sh b/.azure-pipelines/scripts/ut/run_ncoder.sh index 9a13fbb4ce9..2a4c662defc 100644 --- a/.azure-pipelines/scripts/ut/run_ncoder.sh +++ b/.azure-pipelines/scripts/ut/run_ncoder.sh @@ -1,4 +1,22 @@ #!/bin/bash set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" -echo "run coder" \ No newline at end of file +echo "run coder" + +echo "no FWKs need to be installed..." +echo "no requirements need to be installed..." + +cd /neural-compressor/test || exit 1 +find ./neural_coder -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +ut_log_name=${LOG_DIR}/ut_neural_coder.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_ux.sh b/.azure-pipelines/scripts/ut/run_ux.sh index d331242fa6b..86d5a32e44a 100644 --- a/.azure-pipelines/scripts/ut/run_ux.sh +++ b/.azure-pipelines/scripts/ut/run_ux.sh @@ -1,4 +1,29 @@ #!/bin/bash set -x python -c "import neural_compressor as nc;print(nc.version.__version__)" -echo "run ux" \ No newline at end of file +echo "run ux" + +echo "specify fwk version..." +export tensorflow_version='2.9.1' +export pytorch_version='1.11.0+cpu' +export torchvision_version='0.12.0' +export onnx_version='1.9.0' +export onnxruntime_version='1.10.0' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./ux -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index e7561454836..eaf21dd0676 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -43,6 +43,7 @@ stages: exit 1 fi displayName: "Build develop docker image" + - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true @@ -80,6 +81,7 @@ stages: exit 1 fi displayName: "Build develop docker image" + - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true @@ -117,6 +119,7 @@ stages: exit 1 fi displayName: "Build develop docker image" + - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml index dd7dde5c113..8c3195d99d1 100644 --- a/.azure-pipelines/ut-ncoder.yml +++ b/.azure-pipelines/ut-ncoder.yml @@ -11,34 +11,45 @@ pr: pool: suyue-test variables: - PYTHON: 'python3.8' IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'pr' + IMAGE_TAG: 'py38' stages: - stage: displayName: Unit Test for Neural Coder jobs: - job: - variables: - INC_BRANCH: $(System.PullRequest.SourceBranch) - REPO: $(Build.Repository.Uri) steps: - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean Docker' + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + - script: | - echo ${INC_BRANCH} - echo ${REPO} - docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $SYSTEM_DEFAULTWORKINGDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - displayName: 'Build Image' + # for test + docker image rm ${IMAGE_NAME}:${IMAGE_TAG} + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" - script: | - docker images | grep -i neural-compressor - displayName: 'Check docker Image' + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' - script: | - docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_ncoder.sh - displayName: 'Run UT' \ No newline at end of file + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ncoder.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 11d4b025816..93d3e86c3ad 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -11,35 +11,45 @@ pr: pool: suyue-test variables: - PYTHON: 'python3.8' IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'pr' + IMAGE_TAG: 'py38' stages: - stage: displayName: Unit Test for UX jobs: - job: - variables: - INC_BRANCH: $(System.PullRequest.SourceBranch) - REPO: $(Build.Repository.Uri) steps: - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean Docker' + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" - script: | - echo ${INC_BRANCH} - echo ${REPO} - docker build --build-arg PYTHON=${PYTHON} --build-arg REPO=${REPO} --build-arg SOURCE_BRANCH=${INC_BRANCH} -f $BUILD_SOURCESDIRECTORY/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - displayName: 'Build Image' + # for test + docker image rm ${IMAGE_NAME}:${IMAGE_TAG} + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" - script: | - docker images | grep -i neural-compressor - displayName: 'Check docker Image' + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' - script: | - docker run -tid --disable-content-trust --privileged --name="pr-test" --hostname="pr-test-container" ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/.azure-pipelines/scripts/ut/run_ux.sh - displayName: 'Run UT' \ No newline at end of file + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ux.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file From da6045cb25094d3d5b51662509cfc2c091dd254a Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:05:26 +0800 Subject: [PATCH 093/296] log dir path fix --- .azure-pipelines/scripts/ut/run_basic_adaptor.sh | 1 + .azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh | 1 + .azure-pipelines/scripts/ut/run_basic_others.sh | 1 + .azure-pipelines/scripts/ut/run_ncoder.sh | 1 + .azure-pipelines/scripts/ut/run_ux.sh | 1 + 5 files changed, 5 insertions(+) diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index f54f2df1117..334be82959e 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -16,6 +16,7 @@ cd /neural-compressor/test || exit 1 find ./adaptor -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log echo "cat run.sh..." diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh index f0aa8e72ba5..50b11cdf351 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -13,6 +13,7 @@ cd /neural-compressor/test || exit 1 find ./tfnewapi -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut_tf_newapi.log echo "cat run.sh..." diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh index e0e98116548..5cd2e39d1e5 100644 --- a/.azure-pipelines/scripts/ut/run_basic_others.sh +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -22,6 +22,7 @@ sed -i '/ ux\//d' run.sh sed -i '/ neural_coder\//d' run.sh LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log echo "cat run.sh..." diff --git a/.azure-pipelines/scripts/ut/run_ncoder.sh b/.azure-pipelines/scripts/ut/run_ncoder.sh index 2a4c662defc..aef05d13e3f 100644 --- a/.azure-pipelines/scripts/ut/run_ncoder.sh +++ b/.azure-pipelines/scripts/ut/run_ncoder.sh @@ -10,6 +10,7 @@ cd /neural-compressor/test || exit 1 find ./neural_coder -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut_neural_coder.log echo "cat run.sh..." diff --git a/.azure-pipelines/scripts/ut/run_ux.sh b/.azure-pipelines/scripts/ut/run_ux.sh index 86d5a32e44a..ddfff991652 100644 --- a/.azure-pipelines/scripts/ut/run_ux.sh +++ b/.azure-pipelines/scripts/ut/run_ux.sh @@ -17,6 +17,7 @@ cd /neural-compressor/test || exit 1 find ./ux -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut_tf_${tensorflow_version}_pt_${pytorch_version}.log echo "cat run.sh..." From 5bd9b4bc3abb33b6550e8ce990236a4492fb8fd2 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:17:27 +0800 Subject: [PATCH 094/296] fix env setup --- .azure-pipelines/scripts/ut/env_setup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 9be21c9024b..736622e64dc 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -22,6 +22,7 @@ fi if [[ "${onnxruntime}" != "" ]]; then pip install onnxruntime==${onnxruntime_version} + pip install onnxruntime-extensions fi if [[ "${mxnet}" != "" ]]; then @@ -33,9 +34,9 @@ if [ -f "requirements.txt" ]; then sed -i '/^neural-compressor/d' requirements.txt sed -i '/^intel-tensorflow/d' requirements.txt sed -i '/find-links https:\/\/download.pytorch.org\/whl\/torch_stable.html/d' requirements.txt - sed -i '/^torch/d' requirements.txt + sed -i '/^torch/d;/^torchvision/d' requirements.txt sed -i '/^mxnet-mkl/d' requirements.txt - sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt + sed -i '/^onnx/d;/^onnxruntime/d;/^onnxruntime-extensions/d' requirements.txt n=0 until [ "$n" -ge 3 ] do From 8b29c38bbe0ba1d3fef695ec9dace13b25f0694f Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:18:16 +0800 Subject: [PATCH 095/296] fix env setup --- .azure-pipelines/scripts/ut/env_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 736622e64dc..2bc4ea7667b 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -3,7 +3,7 @@ if [[ "${tensorflow_version}" == *"-official" ]]; then pip install tensorflow==${tensorflow_version%-official} elif [[ "${tensorflow_version}" == "spr-base" ]]; then - pip install /tf_dataset/tf_binary/tensorflow*.whl + pip install /tf_dataset/tf_binary/tensorflow*.whl || exit 1 elif [[ "${tensorflow_version}" != "" ]]; then pip install intel-tensorflow==${tensorflow_version} fi From 4fa530e07e34cc56f5604a96c0ff5bc5a85cf586 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:28:42 +0800 Subject: [PATCH 096/296] fix bug --- .azure-pipelines/scripts/ut/env_setup.sh | 5 ++++- .azure-pipelines/ut-basic.yml | 2 -- .azure-pipelines/ut-ncoder.yml | 2 -- .azure-pipelines/ut-ux.yml | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 2bc4ea7667b..46ad39bceb9 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -3,7 +3,10 @@ if [[ "${tensorflow_version}" == *"-official" ]]; then pip install tensorflow==${tensorflow_version%-official} elif [[ "${tensorflow_version}" == "spr-base" ]]; then - pip install /tf_dataset/tf_binary/tensorflow*.whl || exit 1 + pip install /tf_dataset/tf_binary/tensorflow*.whl + if [[ $? -ne 0 ]]; then + exit 1 + fi elif [[ "${tensorflow_version}" != "" ]]; then pip install intel-tensorflow==${tensorflow_version} fi diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index eaf21dd0676..fecf3fb59d2 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -32,8 +32,6 @@ stages: displayName: "Checkout out Repo" - script: | - # for test - docker image rm ${IMAGE_NAME}:${IMAGE_TAG} if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml index 8c3195d99d1..e05add9033e 100644 --- a/.azure-pipelines/ut-ncoder.yml +++ b/.azure-pipelines/ut-ncoder.yml @@ -29,8 +29,6 @@ stages: displayName: "Checkout out Repo" - script: | - # for test - docker image rm ${IMAGE_NAME}:${IMAGE_TAG} if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 93d3e86c3ad..8ebba029a72 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -29,8 +29,6 @@ stages: displayName: "Checkout out Repo" - script: | - # for test - docker image rm ${IMAGE_NAME}:${IMAGE_TAG} if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi From a96019e8037a757e05af9c8cffc0fcde12c61ce5 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:37:59 +0800 Subject: [PATCH 097/296] test only, revert before merge --- neural_coder/__init__.py | 1 + neural_compressor/ux/__init__.py | 1 + 2 files changed, 2 insertions(+) diff --git a/neural_coder/__init__.py b/neural_coder/__init__.py index 46dd8eee68c..304699fab90 100644 --- a/neural_coder/__init__.py +++ b/neural_coder/__init__.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# for test only from .interface import enable from .interface import bench diff --git a/neural_compressor/ux/__init__.py b/neural_compressor/ux/__init__.py index 4dad25e4928..faabe895b72 100644 --- a/neural_compressor/ux/__init__.py +++ b/neural_compressor/ux/__init__.py @@ -12,4 +12,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# for test only """The app package contains all components required for UX server.""" From c05cb2e28cc65e076dbdbff97191c1c15b8e4e87 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 01:41:35 +0800 Subject: [PATCH 098/296] bug fix --- .azure-pipelines/ut-ncoder.yml | 2 +- .azure-pipelines/ut-ux.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml index e05add9033e..e2a60d39a76 100644 --- a/.azure-pipelines/ut-ncoder.yml +++ b/.azure-pipelines/ut-ncoder.yml @@ -4,7 +4,7 @@ pr: branches: include: - master - path: + paths: include: - neural_coder diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 8ebba029a72..dd35dde3896 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -4,7 +4,7 @@ pr: branches: include: - master - path: + paths: include: - neural_compressor.ux From 3456241f38036a7f742e8ee82663cabbe069b411 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 10:22:44 +0800 Subject: [PATCH 099/296] bug fix --- .azure-pipelines/scripts/ut/env_setup.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 46ad39bceb9..45793ce7428 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -19,16 +19,16 @@ if [[ "${torchvision_version}" != "" ]]; then pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html fi -if [[ "${onnx}" != "" ]]; then +if [[ "${onnx_version}" != "" ]]; then pip install onnx==${onnx_version} fi -if [[ "${onnxruntime}" != "" ]]; then +if [[ "${onnxruntime_version}" != "" ]]; then pip install onnxruntime==${onnxruntime_version} pip install onnxruntime-extensions fi -if [[ "${mxnet}" != "" ]]; then +if [[ "${mxnet_version}" != "" ]]; then pip install mxnet==${mxnet_version} fi From c7dac3ada5d477a7256d1c9caf801b881377b6b7 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 11:04:52 +0800 Subject: [PATCH 100/296] echo version --- .azure-pipelines/scripts/ut/env_setup.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 45793ce7428..fa5057bafd4 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -1,4 +1,12 @@ #!/bin/bash +set -x + +echo $tensorflow_version +echo $pytorch_version +echo $torchvision_version +echo $onnx_version +echo $onnxruntime_version +echo $mxnet_version if [[ "${tensorflow_version}" == *"-official" ]]; then pip install tensorflow==${tensorflow_version%-official} @@ -32,6 +40,7 @@ if [[ "${mxnet_version}" != "" ]]; then pip install mxnet==${mxnet_version} fi + cd /neural-compressor/test if [ -f "requirements.txt" ]; then sed -i '/^neural-compressor/d' requirements.txt From 9bf5305473936e0ec230a38c93f4a7c2b9a47601 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 10 Aug 2022 11:21:20 +0800 Subject: [PATCH 101/296] update mxnet install --- .azure-pipelines/scripts/ut/env_setup.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index fa5057bafd4..af008a8bd01 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -36,7 +36,11 @@ if [[ "${onnxruntime_version}" != "" ]]; then pip install onnxruntime-extensions fi -if [[ "${mxnet_version}" != "" ]]; then +if [ "${mxnet_version}" == '1.6.0' ]; then + pip install mxnet-mkl==${mxnet_version} +elif [ "${mxnet_version}" == '1.7.0' ]; then + pip install mxnet==${mxnet_version}.post2 +elif [ "${mxnet_version}" != '' ]; then pip install mxnet==${mxnet_version} fi From 4772070da5ad348f8a42114a859987e61c68caf5 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 12:40:59 +0800 Subject: [PATCH 102/296] code-scan --- .azure-pipelines/code-scan.yml | 158 ++ .../scripts/codeScan/bandit/bandit.sh | 13 + .../scripts/codeScan/copyright/copy_right.sh | 33 + .../scripts/codeScan/pylint/pylint.sh | 29 + .../scripts/codeScan/pyspelling/lpot_dict.txt | 2202 +++++++++++++++++ .../scripts/codeScan/pyspelling/pyspelling.sh | 21 + .../codeScan/pyspelling/pyspelling_conf.yaml | 11 + 7 files changed, 2467 insertions(+) create mode 100644 .azure-pipelines/code-scan.yml create mode 100644 .azure-pipelines/scripts/codeScan/bandit/bandit.sh create mode 100644 .azure-pipelines/scripts/codeScan/copyright/copy_right.sh create mode 100644 .azure-pipelines/scripts/codeScan/pylint/pylint.sh create mode 100644 .azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt create mode 100644 .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh create mode 100644 .azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml new file mode 100644 index 00000000000..93941820799 --- /dev/null +++ b/.azure-pipelines/code-scan.yml @@ -0,0 +1,158 @@ +trigger: + - code-scan + +pr: none + # - master + +pool: docker-agent + +variables: + IMAGE_NAME: 'code-scan' + IMAGE_TAG: '1.0' + VAL_PATH: .azure-pipelines/scripts/codeScan + target_path: "neural_compressor" + BUILD_DIRECTORY: $(Build.SourcesDirectory) + TARGET_BRANCH: $(System.PullRequest.TargetBranch) + +stages: + # - stage: bandit + # displayName: bandit code scan + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # sudo docker system prune || true + # # echo y | docker system prune + # displayName: 'Clean workspace' + + # - checkout: self + # displayName: "Checkout out Repo" + + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' + + # - script: | + # docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" + # displayName: 'bandit code scan' + + + - stage: pylint + displayName: pylint code scan + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" + displayName: 'pylint code scan' + + - stage: pyspelling + displayName: pyspelling code scan + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" + displayName: 'pyspelling code scan' + + - stage: copyright + displayName: copyright code scan + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + displayName: 'copyright code scan' \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh new file mode 100644 index 00000000000..88159ddb8c5 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -0,0 +1,13 @@ +# $1: $(VAL_PATH) +# $2: $(target_path) + +pip install bandit + + +python -m bandit -r -lll -iii $2 > /lpot-bandit.log + +exit_code=$? +if [ ${exit_code} -ne 0 ] ; then + echo "Bandit exited with non-zero exit code."; exit 1 +fi +exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh new file mode 100644 index 00000000000..ce54ab41c97 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -0,0 +1,33 @@ + # $1: $(target_path) + # $2: $(TARGET_BRANCH) - $(System.PullRequest.TargetBranch) + # $3: $(VAL_PATH) + + supported_extensions=(py, sh, yaml) + + set -xe + + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$2) ./$1 > $3/copyright/diff.log + + files=$(cat $3/copyright/diff.log | awk '!a[$0]++') + + for file in ${files} + + do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + echo "Checking license in 1 ${file}" + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> $3/copyright/copyright_issue_summary.log + cat $3/copyright/copyright_issue_summary.log + fi + else + echo "Skipping ${file}" + fi + done + + + ls $3/copyright/copyright_issue_summary.log + exit_code=$? + if [ ${exit_code} -e 0 ] ; then + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + fi + exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh new file mode 100644 index 00000000000..9b17f150f9e --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -0,0 +1,29 @@ +# $1: $(VAL_PATH) +# $2: $(target_path) + +pip install pylint==2.12.1 + +# echo "---------------------" +# pwd +# echo "---------------------" +# ls +# echo "---------------------" +# $1 +# echo "---------------------" +# cd $1 +# ls +# cd .azure-pipelines/scripts/codeScan/pylint +# ls +# echo "---------------------" + + + +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 +# > /lpot-pylint.json +# python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 > $1/pylint/lpot-pylint.json + + exit_code=$? + if [ ${exit_code} -ne 0 ] ; then + echo "PyLint exited with non-zero exit code."; exit 1 + fi + exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt new file mode 100644 index 00000000000..25bf61a1cef --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -0,0 +1,2202 @@ +aa +aac +abc +Abc +AbcAdaptor +AbcTuneStrategy +abi +ABI +absl +abspath +abstractive +acc +Acc +accuracies +acdc +ACDC +Acknowledgement +activations +Adadelta +adam +AdamW +adaptor +Adaptor +AddEmbeddings +AddN +addr +ADDR +AddV +ade +ADE +adresses +AdvProp +ae +aea +af +AGS +ai +ailab +al +albert +alexnet +AlexNet +algo +algos +alibaba +Alibaba +AlignImageChannel +allenai +alloc +ALLREDUCE +alsologtostderr +Aman +amazonaws +amazonlinux +Amodei +AmpConf +AMX +analytics +Analytics +Anastasiia +AnchorGenerator +andravin +andreamad +anisotropic +anno +anton +ap +apache +api +APIs +APl +APm +approch +APs +arg +argmax +ArgMax +args +arxiv +arXiv +asd +astype +asym +async +atrous +att +AttentionReshape +attr +attredirects +AttributeProto +attrs +auc +aug +autgrad +autogenerate +autograd +AutoMixPrecision +autopep +Autoregressive +ava +AverageMeter +avgloss +AvgPool +avx +AVX +backend +backends +backticks +bart +barthez +bashrc +basicConfig +BasicTokenizer +batchgenerators +BatchMatMul +BatchMatMulV +batchnorm +BatchNorm +bayesian +BayesianOptimization +bazel +bbbb +bbox +bboxes +bccf +bce +bd +bdb +bdist +benchmarked +benchmarking +Benchmarking +Bengio +Benoît +berkeleyvision +bert +BERT's +BertAdam +BertConfig +BERTDataSet +BertForMaskedLM +BertForNextSentencePrediction +BertForPreTraining +BertForQuestionAnswering +BertForSequenceClassification +BertForTokenClassification +BertModel +berts +bertsquad +BertTokenizer +bfloat +BFP +BGR +Bianchi +BiasAdd +BibTeX +bicubic +bilibili +BiliBili +bilinear +BilinearImagenet +billsum +BiLSTM +binarize +binarized +BinaryAdd +biomedical +Biomedical +BLAS +blendcnn +BlendCnn +BlendCNN +bleu +BLEU +blocktime +blogpost +bn +bninception +BNInception +bobw +booktitle +bool +BoW +boxlist +BoxList +br +BrainTumour +BraTS +broadcasted +bs +bsnone +bugfix +buildin +builtin +Builtin +BV +bvlcalexnet +bzip +cadene +Cadene +caffe +Caffe +caffenet +Caffenet +cafferesnet +CaffeResnet +CaffeResNet +Caiming +calib +calibrationcsv +camembert +CamemBERT +canada +Carbonell +CascadeFullRes +cbica +cd +cdn +ce +cec +CenterCrop +centernet +centerNet +centos +CentOS +Centre +cern +CERN's +certfile +Cesa +cfa +cffi +cfg +CFLAGS +ChamNet +Chaumond +checkbox +checkboxes +Cheng +chmod +Cho's +chongruo +Chongruo +chris +Chuanqi +ci +cifar +CIFAR +circleci +Cistac +cityscapes +Cityscapes +cityscapesscripts +cityscapesScripts +cknowledge +ckpt +ckpts +ClassPredictionTower +clcarwin +Clergerie +cli +CLI +clipnorm +clm +CLM +cls +CLX +cly +cmake +CMake +CMAKE +CMakeLists +cmd +CMU +cn +cnn +cnt +CoCo +cocoapi +cocoApi +cocodataset +COCODataset +COCODemo +COCOEval +COCOmAP +COCOmAPv +COCONpy +cocoraw +COCORaw +COCORecord +codalab +codecogs +codenamed +CoLA +colorama +ColorJitter +colspan +compat +compilervars +concat +ConcatV +cond +conda +CONDA +condconv +CondConv +Condensenet +conf +config +Config +configs +CoNLL +Conneau +const +ConstantOfShape +ConstDataLoader +constexpr +contaning +conv +Conv +ConvBNReLU +ConvertingSSDMobilenetToONNX +convertion +ConvNets +convolutional +Convolutional +ConvPerStage +ConvReLU +copt +coreml +CoreML +cp +cpp +cpu +cpus +CPUs +CPX +cpython +creafz +creatis +creativecommons +criteo +Criteo +CriteoTerabyte +croping +CropResize +CropToBoundingBox +CrossEntropyLoss +crossvalidaton +crt +csv +ctrl +CTRLModel +CTRLTokenizer +ctuning +ctx +cuda +cudaPopCallConfiguration +cudatoolkit +CUDAToolKit +cudnn +CUHK +curr +Curran +custormer +cv +CvAClvFfyA +CXX +cxxopt +cypw +cython +da +dae +DagnyT +Dai +dailymail +Danqi +darknet +Darknet +datadir +datafile +dataloader +dataLoader +DataLoader +DataLoadermodule +dataloaders +DataParallel +datapoints +DataProcessor +dataset +Dataset +DatasetAnalyzer +datasets +datatype +datatypes +dathath +Dathathri +datset +dbmdz +DBMDZ +dbox +dbs +DCMAKE +dcn +ddp +DDP +DDR +de +deberta +DecodeImage +deepengine +deeplab +DeepLab +deeplabv +DeepLabV +DeepLearningExamples +Delangue +DENABLE +denseblock +denselayer +densenet +DenseNet +deps +DepthwiseConv +dequant +dequantize +DequantizeLinear +DequantStub +DeQuantStub +desc +dest +destructor +detections +detectron +Detectron +dev +devel +Devlin +devtool +DFabiansResUNet +DFS +DGAN +dialogpt +DialoGPT +dicts +dir +dirname +Discrim +distil +Distil +distilbert +DistilBert +DistilBERT +DistilBERT +DistilBertModel +DistilBertTokenizer +distilgpt +DistilGPT +distillated +Distillated +distillating +DistilmBERT +distilrobert +distilroberta +DistilRoBERTa +DistributedDataParallel +DistributedOptimizer +DistributedSampler +distro +dividiti +Djamé +DKFZ +dl +dlabel +dlboost +dlrm +DLRM's +dmlc +DMQA +dNative +dnf +dnn +dnnl +DNNL +Dockerfile +doclist +docstrings +doctrings +docutils +doteq +dowmsampling +downloader +downsampled +downsampling +doxygen +dpn +DPNs +dpr +DropOut +ds +dscore +dst +dtype +DualPathNet +dualpathnetworks +DualPathNetworks +DummyDataLoader +dunet +DUNet +Dupont +Durand +dvdt +dw +dynamiccaly +ead +EAQkaohzrJbd +earlystop +eb +ecotrust +edgetpu +EdgeTPU +edu +ee +eer +ef +efficientnet +efficientNet +EfficientNet +EfficientNets +eg +eightbit +einstein +el +elif +eltwise +emb +embeddings +embs +EMC +enablerepo +EncodeJped +enfr +eng +ensembling +ensp +entrypoint +enum +env +eq +erf +Erf +Éric +eriklindernoren +Errno +esri +et +eval +evaluator +evel +exemplarily +exising +existing +exmaple +expanduser +ExperimentPlanner +ExperimentPlanners +extractive +EzjbRL +fabian +FabiansUNet +facebook +FaceBook +facebookresearch +fairseq +fallbacks +fanout +faq +Farhadi +FashionMNIST +FasterRCNN +FastFormers +fastrcnn +fatihcakirs +favourably +fb +fbgemm +FBNet +fbnetc +fbresnet +FBResNet +fc +fcn +FCN +fd +FeatureExtractor +feedbacks +Feng +ffc +filename +filenames +FileNotFoundError +filepath +filesystem +finbert +finetune +Finetune +finetuned +finetuning +flac +FlatMapDataset +flaubert +flavour +flavours +Flavours +floatfunctional +FloatFunctional +FloatTensor +FLOPs +Florian +fmfn +fmt +fmtstr +fn +fname +fns +foregound +fp +FP +fpic +fPIC +fpn +FPN +FRN +FromConfig +frontend +fstack +ftfy +Fu +fullres +func +functionalities +functionet +functools +Funtowicz +fw +FWK +fx +GameAI +GANs +Garnett +gcc +gclient +gd +geffnet +gelu +Gelu +GeluOperator +GenEfficientNet +GenericPreprocessor +german +germeval +GermEval +gestaltit +getitem +getsize +GetStrides +GFLOPs +gh +gid +Gimpel +Girshick +github +GitHub +githubusercontent +gitmodules +GLIBCXX +GLOG +GLUE +gluebenchmark +gluepy +gluon +Gluon +gluoncv +GluonCV +gluonnlp +gn +GN +goldsborough +goog +google +googleapis +googleblog +googlenet +googlesource +Goyal +gpg +GPG +gpt +gpu +gpus +GPUs +graphdef +GraphDef +GraphModule +GraphProto +Grauman +grpc +gtFile +gtFine +Gui +Guillaume +Guoming +gz +gzY +Haibin +haibinlin +Haihao +hangzhang +hardcoding +HasAns +hawq +HAWQ +HdQ +heatmaps +Hein +helloworld +HelloWorld +henson +hiddenlayer +hippocampus +Hippocampus +HistogramObserver +hlu +horovod +Horovod +HOROVOD +horovodrun +hostfile +Hounsfield +howpublished +HqEgzS +href +html +http +https +Hu +hubert +huggingface +HuggingFace +HuggingFace's +HuggingFacesTS +hujie +hvd +HybirdBlock +HybridBlock +hyperparameter +hyperparameters +icc +ICCV +Icelake +icpc +icx +ide +idx +ie +IEEE +ILSVR +ilsvrc +ILSVRC +Ilya +im +imagecocodataset +ImageFolder +ImageList +imagenet +ImageNet +ImagenetRaw +ImageRecord +ImageRecordIter +imagesTr +imagesTs +img +imgrec +imgs +imgx +IML +impl +ImportError +IMS +inceptionresnetv +InceptionResNetV +inceptionv +InceptionV +incollection +IndexType +indexValue +indices +indico +inferencer +informations +infos +init +InnerProduct +innersource +inp +inplace +inproceedings +inputcsv +InputData +InputExample +InputFile +Inria +insa +instanceonly +instantiation +integerops +intel +intelai +IntelAI +interoperability +introudces +ints +inturn +InvertedResidual +io +ios +iOS +iou +IoU +ipc +ipex +IPEX +ipynb +ipython +ir +irv +ISA +Isensee +isinstance +issuecomment +IssueQuery +IssueQueryThreads +iter +IteratorGetNext +iters +Jäger +jemalloc +Jens +Jie +jim +Jingfei +Jiong +jit +jitter +Joshi +jpeg +JPEGImages +jpg +jpwarren +json +jsons +Julien +JunWang +jupyter +kaggle +kaggleAdDisplayChallenge +kaiming +KaimingHe +Karthik +kcho +keepbs +keepdim +keras +Keskar +keyfile +keypoint +Keypoint +kimiyoung +kitti +kmp +KMP +KnowledgeDistillationLoss +kriz +kwargs +Kyunghyun +LabelShift +labelsTr +Lample +Lan +lang +LanguageModeling +Lapata +Larochelle +LastLayerShape +latencies +LaTeX +Lavin +layernorm +LayerNorm +layoutlm +ld +len +LessEqual +lf +lfaidata +lfs +li +libdeep +libengine +libffi +libGL +libglib +libiomp +libmlperf +librispeech +LibriSpeech +librosa +libsndfile +libstdc +libz +licence +liKE +Limitting +lin +linkopt +linoxide +linux +linuxfoundation +ListDataset +LiTS +Liu +Liu's +llvmlite +lm +LMHeadModel +ln +loadgen +LoadGen +LOADGEN +LoadGen's +LoadgenAPI +LoadgenAPITestSettings +LoadgenVersion +LoadImage +LOC +localdisk +localhost +LOCderiv +LOCpart +logdir +logfile +login +logits +LOGLEVEL +LogSettings +logtostderr +longformer +lossy +Louf +LowPrecisionInferenceTool +lowproposals +lowres +Lp +lpot +LPOT +LPOT's +lr +lS +LSVRC +lt +LTS +lua +Luan +lutzroeder +lyon +Lysandre +lzma +macOS +MACOSX +MAdds +Madotto +MagnitudePrunePolicy +Maier +mainpage +Makefile +MakefileGnProj +MakeIterator +Mandar +Manmatha +manylinux +mAp +mAP +Mapillary +marianmt +MaskPostProcessor +maskrcnn +MaskRCNN +MaskRCNNFPNFeatureExtractor +maskrnn +massa +Massa +matcher +matmul +MatMul +MatMulWithBias +MatMulWithBiasAdd +MatMulWithBiasGelu +MatMulWithBiasTanh +matplotlib +matricses +maxdepth +maxindrange +maxk +MaxPool +maxSizeInComplete +mbart +mBERT +mcc +McCann +mcordts +md +MeanSquaredError +measurer +Medcial +medicaldecathlon +meetup +mem +membind +mems +messi +metabuild +metadata +metamind +MICCAI +microsoft +miguelgrinberg +Mingda +minibatch +minilm +minimalistic +minival +minloglevel +minmax +MinMaxObserver +mins +mIoU +mIOU +Mirella +misalignments +miscs +Mish +missmatches +MixedConv +mixnet +MixNet +mixup +mkdir +mkl +MKL +mlap +mlas +MLAS +mlcommons +mll +mlm +mlp +mlpc +mlperf +MLperf +MLPerf +mlt +mmdetection +mmlab +MMLAB +mnasnet +MNASNet +mnist +MNIST +mnli +MNLI +mobilebert +MobileBERT +mobilenet +MobileNet +mobilenetv +Mobilenetv +MobileNetv +MobileNetV +modalities +Modalities +modality +ModelConversion +modelfeatures +modelforward +modelinput +modellogits +modelmean +modelsize +modelstd +ModuleDict +ModuleNotFoundError +Molino +mpi +mrcnn +mrpc +MRPC +MSD +mse +MSE +msvc +mul +mult +multi +Multi +multiclass +multilabel +multinli +MultiNLI +multiscale +MULTISCALE +MultiStream +MultiStream's +MultiStreamFree +mutli +mv +mx +mxnet +MxNet +MXNet +MyDataset +Mykhailo +Myle +MyMetric +myModel +MYTASK +MYTASKNAME +Naman +namedtuple +nanohanno +Narasimhan +NAS +nasnet +NASNet +nasnetalarge +nasnetamobile +nb +nbest +nbsp +nc +NCCL +nchw +NCHW +nd +ndarray +NDArray +nderlu +nepoch +ner +NER +nervanasystems +nesterov +NetEase +netron +Netron +networkbuilders +NeurIPS +neval +NewMetric +newstest +nextplatform +ng +ngatang +NGPUS +ngram +NHWC +NIC +nifti +niftis +nii +Nijmegen +Nitish +nl +NLG +nli +nll +nlp +NLP +nlpyang +nltk +NLU +nm +nms +nn +nnodes +nnu +nnU +nnunet +nnUnet +nnUNet +nnUNetPlansv +nnUNetTrainer +nnUNetTrainers +nnUNetTrainerV +NNZ +noduplicates +NoisyStudent +Nonlinearity +NonNestedTuple +NoNormalization +NonZero +noobj +np +nproc +npy +npz +nq +nrix +ns +nsdf +nSsKchNAySU +nthreads +ntrain +num +numactl +numba +numCompleteThreads +numerics +numpy +numTest +numTraining +NVAITC +nvcc +nvidia +NVIDIA +NVIDIA's +nvme +Nx +nyu +ok +ol +Omer +OMP +onboarding +oneapi +oneAPI +onednn +oneDNN +onlinedocs +onnx +ONNX +ONNXQuantizer +onnxrt +ONNXRT +onnxruntime +OnnxRuntime +oob +OOM +OOQtYMH +openai +OpenAI +OpenAI's +OpenAIAdam +OpenAIGPTModel +OpenAIGPTTokenizer +opencv +OpenCV +openmp +openslr +opensource +openssl +openvino +OpenVINO +openvinotoolkit +OpenWebTextCorpus +OperatorConfig +OPs +opset +opsetid +optim +optimizations +Optimizations +optimizers +Optimizers +optypewise +opwise +OrderedDict +ORGderiv +ORGpart +os +osJJ +OTH +OTHderiv +OTHpart +Ott +oup +outdir +OutputData +outputfile +ov +overfeat +overfit +overfitted +PaddingSequence +PaddingSequence +pageId +palletsprojects +panoptic +Panoptic +paperswithcode +param +params +Parinov +ParseDecodeImagenet +ParseDecodeVoc +participations +Parzen +pastebin +patientIDs +pb +pbar +pdf +Peason +pegasus +pelee +peleenet +PeleeNet +Penghui +Pengxin +pepy +PerChannelMinMaxObserver +PERderiv +perf +perftests +PERpart +phrasebank +phy +physcpubind +PhYUmn +Piero +Pierric +PIL +pixAcc +Piyush +pjreddie +pkill +pkl +pky +plm +PLM +pls +pnasnet +PNASNet +png +POC +polynet +PolyNet +Pooler +pos +postprocesing +postprocess +postprocessed +postprocessing +PostProcessor +PostTransform +PowerTools +pplm +PPLM +PQ +pre +prebuild +prebuilt +Prec +precisions +pred +preds +preformance +Preload +preprint +preprocess +preprocessed +preprocesses +preprocessing +preprocessor +Preprocessor +PreprocessorFor +Preprocessors +prerelease +PreSumm +pretrain +pretrained +pretrainedmodels +pretraining +prev +prioritizies +probs +proc +productizing +profilings +ProgressBar +proto +Protobuf +protoc +protractortest +PRs +PrunePolicy +pth +ptq +PTQ +ptr +pudae +pw +PWC +pwd +PWD +px +py +pybind +pycocotools +pyguide +pylint +pymodule +PyObject +pypi +PyPI +PySUT +pytest +PythonAPI +PYTHONPATH +pytorch +PyTorch +pytorchic +PyTorchKnowledgeDistillationLoss +pyyaml +PyYAML +PZ +qat +QAT +qconfig +QConfig +QiaoranC +qint +qlinear +QLinear +qlinearops +QnA +qnli +QNLI +qps +QPS +qqp +QQP +qscheme +qsl +QSL +qtcreator +qtype +quant +quantile +quantizable +Quantizable +quantization +Quantization +quantize +quantized +QuantizedConv +QuantizedConvReLU +QuantizedInput +quantizer +quantizes +Quantizes +quantizing +QuantStub +QueryBackendCapability +QuerySampleComplete +QuerySampleLibrary +quickstart +Quickstart +QuickStart +Quoc +R'emi +Radboud +Radford +Radu +rAjHyXhTzz +rajpurkar +ramdisk +RandAug +RandAugment +randn +RandomCrop +RandomHorizontalFlip +RandomResizedCrop +RandomVerticalFlip +Rault +rc +rcnn +readme +README +ReadmeBuild +ReadmeFAQ +ReadmeHtmlDocs +ReadmeTests +readthedocs +realtime +Realtime +rebase +recommonmark +RecordingObserver +recordio +RecordIO +recurse +Redmon +ReduceMean +regex +RegNet +rehm +Rehm +reinstall +relase +relu +Relu +ReLU +repo +repo's +repo’s +repos +representating +requantize +resampled +resampling +rescale +Rescale +ResencUNet +resize +Resize +ResizeCropImagenet +resized +Resizes +ResizeWithRatio +resnest +ResNest +ResNeSt +resnet +Resnet +ResNet +resnetv +ResNetV +resnext +ResNext +ResNeXt +ressource +ressources +reStructuredText +ret +RetinaMask +retinanet +retinaNet +RetinaNet +reusability +Rewon +rf +rfcn +rgb +RGB +rmax +rmin +RMSE +rn +rng +RNN +rnnt +ro +roberta +RoBERTa +RobertaModel +RobertaTokenizer +ROC +RocStories +Romary +rosanneliu +rougeL +rougeLsum +rowanz +rowspan +RPN +RPNHead +RPNPostProcessor +Rsqrt +rst +rtd +RTX +runhooks +runtime +Runtime +RuntimeError +Rusia +rusiaaman +Ruslan +rw +rwightman +sacremoses +Sagot +Salakhutdinov +salesforce +Salesforce +Salimans +sanh +Sanh +sata +SavedModel +SavedModel +Scalable +scaler +scatterFillKernel +sched +scikit +scm +screenshots +ScriptModule +se +sed +Seddah +seg +segm +SegmentationMask +segmentations +seid +senet +SENet +sentencepiece +Sep +SEP +SeqDataCollator +serializable +ServerPool +sess +setuptools +sexualized +SGD +sgmoid +SHA +sharded +Sharma +Shen +Shirish +shouldn +showEvent +shufflenet +Shufflenet +ShuffleNet +shufflenetv +Shvets +sigmoid +signup +sigopt +Sigopt +SigOpt +SingleStream +skx +Skylake +skylion +SMBO +SMBOs +Smola +smoothes +sndfile +Socher +socio +SocketIO +softmax +somain +Soricut +sota +SOTA +sox +SoX +spacings +spacy +SpaCy +SparseCategoricalAccuracy +SparseCategoricalCrossentropy +sparsified +Spearman +spearmanr +specificities +splitted +spm +spnasnet +sqlalchemy +Sqrt +sqSiUy +Squad +SQuAD +SquadF +squadpy +SquadV +SquaredDifference +squeezebert +squeezenet +SqueezeNet +src +SrcTuple +sryqufw +ssd +SSDMobilenet +SSDSC +sshleifer +sst +stackoverflow +Standley +startswith +StartTest +stdout +stds +stefan +stemblock +stepsize +Stoyanov +str +strided +struct +sts +STS +stsb +styleguide +Suárez +subexpression +subfolder +subfolders +Subgraph +submodule +submodules +Submodules +subsample +subtoken +sudo +Sumanth +summarization +Summarization +SummaryWriter +superseeds +suported +sut +SUT +Sutskever +sv +svg +swagaf +sym +symlink +symlinked +symlinks +Symlinks +synset +sys +SystemUnderTest +tanh +TaskXX +TaskXXX +tb +TBD +tbe +tbody +td +techdecoded +tencent +tensor's +tensorboard +tensorBoard +TensorBoard +tensorcore +TensorDataset +tensorflow +TensorFlow +TensorflowQuery +tensorImageSize +TensorInfo +TensorProto +teraoperations +tesla +testability +TestSettings +tf +TF +TFBertForSequenceClassification +tflite +tfp +tfrecord +TFRecord +TFRecordDataset +tfrecords +TFRobertaModel +TFSlimNetsFactory +TFSlimNetsFactory's +tg +tgt +tgz +th +THCudaTensor +thead +thepath +thres +thrs +Tian +Tidx +timeline +timestamps +TinyBERT +tl +tlkh +tLoss +TLS +tmp +tmpfs +ToArray +ToBGR +toc +toctree +TODO +tokenization +tokenize +tokenized +tokenizer +Tokenizer +tokenizers +Tokenizers +tokenizing +tol +TOL +tolist +toml +ToNDArray +toolchains +ToPILImage +topk +TopK +topologies +ToRange +torchaudio +torchscript +TorchScript +torchtext +torchvision +TorchVision +toronto +totalizing +ToTensor +Toutanova +tp +tpe +TPE +tpu +TPU +tqdm +traceback +trainings +trainval +trainvaltest +transfo +TransformImage +TransfoXLModel +TransfoXLTokenizer +travis +trigram +tstandley +tsv +TuneStrategy +tunings +tuningusage +tuple +tuples +txt +TZ +uber +ubuntu +ubyte +UI +UID +uint +uk +un +uncomment +uncompress +unet +Unet +UNet +unidecode +uniq +unittest +unref +unsqueeze +unstack +upenn +uploader +upscaled +Upscaled +upstreamed +url +userspace +usp +usr +UTC +util +utils +ux +UX +valminusminival +valset +ValueError +Varshney +VCVTNE +VCVTNEPS +VDPBF +vec +Veronika +veronikayurchuk +versioned +Veselin +vgg +viewpage +Villemonte +ViT +voc +VOC +VOCdevkit +VOCmAP +VOCMApMetrics +VOCRecord +voxel +voxels +vram +VRAM +VTune +waleedka +Wallach +wangg +warmup +wav +wd +webcam +Webcam +webite +webpage +WebSockets +WebText +wedam +WeightSharedConvolutionalBoxPredictor +Wformat +wget +whitelist +whl +WideResNet +WideResNet +Wightman +wikipedia +wikitext +WikiText +WilsonCity +WIP +WLYDCRB +wmt +wnd +WnD +wnli +Wnxu +WordPiece +workdir +workflow +Workflow +workflows +workspace +wrt +wwm +www +xad +xception +Xception +xcode +xeon +Xeon +Xiang +Xiong +xl +XLA +xlm +XLMModel +XLMTokenizer +xlnet +XLNet +XLNetModel +XLNetTokenizer +XlUH +xml +xnli +XNLI +xsum +xV +xvf +xvzf +XXXX +xxy +xxz +xYNrZdEAnrHk +xywh +xyxy +xz +xzvf +yacs +yaml +yamls +Yi +Yiming +Yinhan +yizhu +yjxiong +YKd +Yoann +yolo +yolov +YOLOv +YOLOV +yosinski +Yosinski +YqgzY +Yuanjun +Yue +Yunpeng +Yurchuk +YY +zenodo +Zettlemoyer +zfnet +ZFNet +zh +zhang +Zhang +zhanghang +Zhenzhong +Zhi +Zhilin +Zhongyue +zhongyuezhang +Zhu +Zihang +zihangdai +znoexecstack +znow +Zptls +zrelro +zrl +zxvf +CustomObj +ModelSize +QDQ +QLinearOps +qdq +qdqops +CodeGenerator +GEMM +SparseLib +Xbyak +brgemm +cfgs +gtests +hpp +hypotype +kd +ker +kern +sparsednn +spmm +xxxx +GraphModules +wsl +descs +gtest +IOMP +MALLOC +PRETAINED +SPR +libjemalloc +preload +thp +GCP +gcp +gif +solutionslibrary +geomean +VNNI +Preloading +DUC +duc +leftImg +roc +sklearn +CLA +cla +whitehat +codeofconduct +CYP +SBSTD +xd +samsum +IntelCaffe +baremetal +HWs +IC +KH +NVidia +OC +bolded +sparsification +tensorrt +hardwares +BenchmarkConf +PruningConf +DistillationConf +grey +ModelZoo +mzbert +CaffeNet +FlauBERT +GoogleNet +SqueezeBERT +iz +lvwerra +mBart +oje +za +zk +QIntegerops +QLinearops +criterions +HuBERT +csarron +gpb +howey +huawei +noah +nreimers +pruneofa +textattack +scheduler's +BiDAF +bidaf +FERPlus +ferplus +MixedPrecision +DUnetCNN +calibrationset +ndhwc +ArcFace +arcface +arcfaceresnet +nfolds +RFB +WIDERFACE +shuoyang +ultraface +XKeyboard +lscpu +qpa +vnni +xcb +DevCloud +PyPi +aidevcloud +awk +clx +devcloud +lAtr +nda +ppn +qstat +qsub +qsvr +ruserok +scp +spr +stderr +uXXXXX +QuantConf +SuperBench +autocast +kai +mailto +superbench +yao +Lecun +NLPToolkit +Yan +exdb +lecun +publis +yann +abcadf +bcb +INTRA +WARMUPS +ende +intra +inteltensorflow \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh new file mode 100644 index 00000000000..26535ae9d23 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -0,0 +1,21 @@ +# $1: $(VAL_PATH) +# $2: $(BUILD_DIRECTORY) - $(Build.SourcesDirectory) + +pip install pyspelling + +pip install aspell + +pip install aspell-en + +sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" $1/pyspelling/pyspelling_conf.yaml +echo "Modified config:" +cat $1/pyspelling/pyspelling_conf.yaml +pyspelling -c $1/pyspelling/pyspelling_conf.yaml > $1/pyspelling/pyspelling_output.log +exit_code=$? +if [ ${exit_code} -ne 0 ] ; then + echo "Pyspelling exited with non-zero exit code."; exit 1 +fi +exit 0 + + diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml new file mode 100644 index 00000000000..c407f846ea2 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml @@ -0,0 +1,11 @@ +matrix: +- name: Markdown + hunspell: + d: en_US.ISO8859-15 + dictionary: + wordlists: + - ${VAL_REPO}/pyspelling/lpot_dict.txt + output: ${VAL_REPO}/pyspelling/lpot_dict.dic + sources: + - ${LPOT_REPO}/neural_coder/**/*.md + - ${LPOT_REPO}/neural_coder/*.md \ No newline at end of file From 0f1b9a9308e89711ed5cb84ce26ac6b40e05a99d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 12:48:43 +0800 Subject: [PATCH 103/296] test --- .azure-pipelines/code-scan.yml | 68 ++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 93941820799..de151e86595 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,51 +9,53 @@ pool: docker-agent variables: IMAGE_NAME: 'code-scan' IMAGE_TAG: '1.0' - VAL_PATH: .azure-pipelines/scripts/codeScan + VAL_PATH: /.azure-pipelines/scripts/codeScan target_path: "neural_compressor" BUILD_DIRECTORY: $(Build.SourcesDirectory) TARGET_BRANCH: $(System.PullRequest.TargetBranch) stages: - # - stage: bandit - # displayName: bandit code scan - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # sudo docker system prune || true - # # echo y | docker system prune - # displayName: 'Clean workspace' + - stage: bandit + displayName: bandit code scan + dependsOn: [] + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # sudo docker system prune || true + # echo y | docker system prune + displayName: 'Clean workspace' - # - checkout: self - # displayName: "Checkout out Repo" + - checkout: self + displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' - # - script: | - # docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" - # displayName: 'bandit code scan' + - script: | + docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" + displayName: 'bandit code scan' - stage: pylint displayName: pylint code scan + dependsOn: [] jobs: - job: Build displayName: Build @@ -89,6 +91,7 @@ stages: - stage: pyspelling displayName: pyspelling code scan + dependsOn: [] jobs: - job: Build displayName: Build @@ -124,6 +127,7 @@ stages: - stage: copyright displayName: copyright code scan + dependsOn: [] jobs: - job: Build displayName: Build From 370e449c5c57c19273b1c341e9f03e6aef8b38cc Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:04:31 +0800 Subject: [PATCH 104/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index de151e86595..f8be5397687 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -127,7 +127,7 @@ stages: - stage: copyright displayName: copyright code scan - dependsOn: [] + dependsOn: [] jobs: - job: Build displayName: Build From db2c237e3437b8c472c42f92b72c5853891dea3a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:06:56 +0800 Subject: [PATCH 105/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f8be5397687..dc506458179 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,7 +9,7 @@ pool: docker-agent variables: IMAGE_NAME: 'code-scan' IMAGE_TAG: '1.0' - VAL_PATH: /.azure-pipelines/scripts/codeScan + VAL_PATH: /neural_compressor/.azure-pipelines/scripts/codeScan target_path: "neural_compressor" BUILD_DIRECTORY: $(Build.SourcesDirectory) TARGET_BRANCH: $(System.PullRequest.TargetBranch) From 3c008bc2559e08132a12e37f0e73e235a642d699 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:10:47 +0800 Subject: [PATCH 106/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 5 ++++- .azure-pipelines/scripts/codeScan/pylint/pylint.sh | 2 ++ .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index ce54ab41c97..56aecc99d67 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -1,6 +1,9 @@ # $1: $(target_path) # $2: $(TARGET_BRANCH) - $(System.PullRequest.TargetBranch) - # $3: $(VAL_PATH) + # $3: $(VAL_PATH) + + pip install -r requirements.txt + supported_extensions=(py, sh, yaml) diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 9b17f150f9e..7c0c04a3d39 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -2,6 +2,8 @@ # $2: $(target_path) pip install pylint==2.12.1 +pip install -r requirements.txt + # echo "---------------------" # pwd diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 26535ae9d23..84f01e5bd1e 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -6,6 +6,8 @@ pip install pyspelling pip install aspell pip install aspell-en +pip install -r requirements.txt + sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling/pyspelling_conf.yaml sed -i "s|\${LPOT_REPO}|.|g" $1/pyspelling/pyspelling_conf.yaml From e19db8fabe3913d376711b07b192c9011bb2e48f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:17:17 +0800 Subject: [PATCH 107/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 56aecc99d67..25716d43416 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe - git --no-pager diff --name-only $(git show-ref -s remotes/origin/$2) ./$1 > $3/copyright/diff.log + git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat $3/copyright/diff.log | awk '!a[$0]++') From 322f84772925c27af30661a26253bdc4f5213176 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:22:07 +0800 Subject: [PATCH 108/296] test --- .../scripts/codeScan/copyright/copy_right.sh | 11 ++++++++++- .../scripts/codeScan/pylint/pylint.sh | 15 --------------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 25716d43416..dcc571d1db4 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,16 @@ set -xe - git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log + echo "------------------" + $1 + echo "------------------" + $2 + echo "------------------" + $3 + echo "------------------" + + git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) ./$1 > $3/copyright/diff.log + # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat $3/copyright/diff.log | awk '!a[$0]++') diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 7c0c04a3d39..e6cdeba1893 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -5,21 +5,6 @@ pip install pylint==2.12.1 pip install -r requirements.txt -# echo "---------------------" -# pwd -# echo "---------------------" -# ls -# echo "---------------------" -# $1 -# echo "---------------------" -# cd $1 -# ls -# cd .azure-pipelines/scripts/codeScan/pylint -# ls -# echo "---------------------" - - - python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 # > /lpot-pylint.json # python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 > $1/pylint/lpot-pylint.json From 9bfc38a97ba9b1183024999cf933f0430f5ba019 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:25:03 +0800 Subject: [PATCH 109/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index dcc571d1db4..d67cfe5490e 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,14 +9,6 @@ set -xe - echo "------------------" - $1 - echo "------------------" - $2 - echo "------------------" - $3 - echo "------------------" - git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) ./$1 > $3/copyright/diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log From 574c5aa2cfc204eed4b7130aaffb22e41e9c77e0 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:29:22 +0800 Subject: [PATCH 110/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index d67cfe5490e..6bc4872e264 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe - git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) ./$1 > $3/copyright/diff.log + git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > $3/copyright/diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat $3/copyright/diff.log | awk '!a[$0]++') From 105427f891c11a362cc48ada7e9c79ed7fb833e8 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:32:27 +0800 Subject: [PATCH 111/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 6bc4872e264..a97ce79d704 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe - git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > $3/copyright/diff.log + git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat $3/copyright/diff.log | awk '!a[$0]++') From 56256ac074fff0fccfb87b01495e02878b2df53f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:33:01 +0800 Subject: [PATCH 112/296] test --- .azure-pipelines/code-scan.yml | 188 ++++++++++++++++----------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index dc506458179..0677999ea74 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -15,115 +15,115 @@ variables: TARGET_BRANCH: $(System.PullRequest.TargetBranch) stages: - - stage: bandit - displayName: bandit code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # sudo docker system prune || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - stage: bandit + # displayName: bandit code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # sudo docker system prune || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' - - script: | - docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" - displayName: 'bandit code scan' + # - script: | + # docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" + # displayName: 'bandit code scan' - - stage: pylint - displayName: pylint code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - stage: pylint + # displayName: pylint code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' - - script: | - docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" - displayName: 'pylint code scan' + # - script: | + # docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" + # displayName: 'pylint code scan' - - stage: pyspelling - displayName: pyspelling code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - stage: pyspelling + # displayName: pyspelling code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' - - script: | - docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" - displayName: 'pyspelling code scan' + # - script: | + # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" + # displayName: 'pyspelling code scan' - stage: copyright displayName: copyright code scan From 60a5cfc0a62cac35b489df51e7d1883d5c5e49f4 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:35:56 +0800 Subject: [PATCH 113/296] test --- .../scripts/codeScan/copyright/copy_right.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index a97ce79d704..8b22408efc1 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -12,7 +12,8 @@ git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log - files=$(cat $3/copyright/diff.log | awk '!a[$0]++') + files=$(cat /diff.log | awk '!a[$0]++') + # files=$(cat $3/copyright/diff.log | awk '!a[$0]++') for file in ${files} @@ -20,8 +21,8 @@ if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then echo "Checking license in 1 ${file}" if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - echo ${file} >> $3/copyright/copyright_issue_summary.log - cat $3/copyright/copyright_issue_summary.log + echo ${file} >> /copyright_issue_summary.log + cat /copyright_issue_summary.log fi else echo "Skipping ${file}" @@ -29,7 +30,7 @@ done - ls $3/copyright/copyright_issue_summary.log + ls /copyright_issue_summary.log exit_code=$? if [ ${exit_code} -e 0 ] ; then echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 From 173e8aa448fc65510e323f7862702538ac87e148 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:39:19 +0800 Subject: [PATCH 114/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 8b22408efc1..bfb387c94e4 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe - git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From b504de863c94925ed14ce75e148367c615def456 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:42:38 +0800 Subject: [PATCH 115/296] tst --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index bfb387c94e4..7c0ee7ec603 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -8,8 +8,8 @@ supported_extensions=(py, sh, yaml) set -xe - - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log + git show-ref -s remotes/origin$2 + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin$2) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From 40a375d7b2adcb4be677fdd1bfc99414371aed9f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:44:21 +0800 Subject: [PATCH 116/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 7c0ee7ec603..3f8a92e39e3 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,8 @@ set -xe git show-ref -s remotes/origin$2 - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin$2) /neural_compressor/neural_compressor > /diff.log + git --no-pager diff --name-only $(git show-ref -s remotes/origin/master ./neural_coder/) > /diff.log + # git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin$2) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From 08b489761a2c11dbeded5e74eb41b531d94ab80b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:50:12 +0800 Subject: [PATCH 117/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 3f8a92e39e3..dcb2ffeb33f 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -8,9 +8,8 @@ supported_extensions=(py, sh, yaml) set -xe - git show-ref -s remotes/origin$2 - git --no-pager diff --name-only $(git show-ref -s remotes/origin/master ./neural_coder/) > /diff.log - # git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin$2) /neural_compressor/neural_compressor > /diff.log + git config --global --add safe.directory /neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin$2) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From 2389bb54413dbdf764a9bcf0cf74fb7b60dfbd84 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:53:35 +0800 Subject: [PATCH 118/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index dcb2ffeb33f..33355341b68 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe git config --global --add safe.directory /neural_compressor - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin$2) /neural_compressor/neural_compressor > /diff.log + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From 1a1533a3a7601902a79dff33dc7954920c87c01b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 13:59:33 +0800 Subject: [PATCH 119/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 33355341b68..0766fb5ef0c 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe git config --global --add safe.directory /neural_compressor - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/pyspelling) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From 39dfeaae3b7b4977273b203a803746d5c5f862ac Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 14:01:47 +0800 Subject: [PATCH 120/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 0766fb5ef0c..33355341b68 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,7 +9,7 @@ set -xe git config --global --add safe.directory /neural_compressor - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/pyspelling) /neural_compressor/neural_compressor > /diff.log + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log files=$(cat /diff.log | awk '!a[$0]++') From fc59acc1bacee042768e36e7d3695fbf51f0dc38 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 10 Aug 2022 14:03:06 +0800 Subject: [PATCH 121/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 33355341b68..7610df5a4c9 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,6 +9,8 @@ set -xe git config --global --add safe.directory /neural_compressor + git config --global user.email xuehao.sun@intel.com + git config --global user.password 271828Sxh! git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log From e35b1ec20dc4c67196debf92720c83a2688d17ce Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 10 Aug 2022 14:04:40 +0800 Subject: [PATCH 122/296] test --- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 7610df5a4c9..33355341b68 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -9,8 +9,6 @@ set -xe git config --global --add safe.directory /neural_compressor - git config --global user.email xuehao.sun@intel.com - git config --global user.password 271828Sxh! git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log From a963a27f4aedb680e36fc32404f1378068dfea6a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 14:42:13 +0800 Subject: [PATCH 123/296] test --- .azure-pipelines/code-scan.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 0677999ea74..29d5c72b7d6 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,8 +1,11 @@ trigger: - code-scan -pr: none - # - master +pr: + - master + +# pr: none +# # - master pool: docker-agent From 438766efbf76e5f84d5adce24b9e494fb60a7112 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 15:01:51 +0800 Subject: [PATCH 124/296] test --- .azure-pipelines/code-scan.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 29d5c72b7d6..9c69695d505 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -161,5 +161,6 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) + # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" displayName: 'copyright code scan' \ No newline at end of file From 201f47c168ad8d9b786f49937bf014a1414b6802 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 15:05:14 +0800 Subject: [PATCH 125/296] test --- .azure-pipelines/code-scan.yml | 206 +++++++++--------- .../scripts/codeScan/pylint/pylint.sh | 14 +- .../scripts/codeScan/pyspelling/pyspelling.sh | 2 - 3 files changed, 109 insertions(+), 113 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9c69695d505..8ad6d6904a8 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -18,115 +18,115 @@ variables: TARGET_BRANCH: $(System.PullRequest.TargetBranch) stages: - # - stage: bandit - # displayName: bandit code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # sudo docker system prune || true - # # echo y | docker system prune - # displayName: 'Clean workspace' + - stage: bandit + displayName: bandit code scan + dependsOn: [] + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # sudo docker system prune || true + # echo y | docker system prune + displayName: 'Clean workspace' - # - checkout: self - # displayName: "Checkout out Repo" + - checkout: self + displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - - # - script: | - # docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" - # displayName: 'bandit code scan' - - - # - stage: pylint - # displayName: pylint code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # echo y | docker system prune - # displayName: 'Clean workspace' + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" + displayName: 'bandit code scan' + + + - stage: pylint + displayName: pylint code scan + dependsOn: [] + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + displayName: 'Clean workspace' - # - checkout: self - # displayName: "Checkout out Repo" + - checkout: self + displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - - # - script: | - # docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" - # displayName: 'pylint code scan' - - # - stage: pyspelling - # displayName: pyspelling code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # echo y | docker system prune - # displayName: 'Clean workspace' + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" + displayName: 'pylint code scan' + + - stage: pyspelling + displayName: pyspelling code scan + dependsOn: [] + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + displayName: 'Clean workspace' - # - checkout: self - # displayName: "Checkout out Repo" + - checkout: self + displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - - # - script: | - # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" - # displayName: 'pyspelling code scan' + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" + displayName: 'pyspelling code scan' - stage: copyright displayName: copyright code scan diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index e6cdeba1893..1afd7f7e975 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -6,11 +6,9 @@ pip install -r requirements.txt python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 -# > /lpot-pylint.json -# python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 > $1/pylint/lpot-pylint.json - - exit_code=$? - if [ ${exit_code} -ne 0 ] ; then - echo "PyLint exited with non-zero exit code."; exit 1 - fi - exit 0 \ No newline at end of file + +exit_code=$? +if [ ${exit_code} -ne 0 ] ; then + echo "PyLint exited with non-zero exit code."; exit 1 +fi +exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 84f01e5bd1e..e4595a69e61 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -2,9 +2,7 @@ # $2: $(BUILD_DIRECTORY) - $(Build.SourcesDirectory) pip install pyspelling - pip install aspell - pip install aspell-en pip install -r requirements.txt From 757fe09933f21b2a3f0b3452540a85cf548c2743 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Wed, 10 Aug 2022 15:42:20 +0800 Subject: [PATCH 126/296] Update code-scan.yml for Azure Pipelines --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 8ad6d6904a8..9ca2b52121a 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -7,7 +7,7 @@ pr: # pr: none # # - master -pool: docker-agent +pool: suyue-test variables: IMAGE_NAME: 'code-scan' From 7cc923743ae54067c873722f96abe244ce74fc72 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 15:48:25 +0800 Subject: [PATCH 127/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9ca2b52121a..3cc877f1b99 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -161,6 +161,6 @@ stages: displayName: 'Clean Docker' - script: | - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" displayName: 'copyright code scan' \ No newline at end of file From 2e0719b26facbc175f44bc2793749f2ed5c68b0d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 15:53:32 +0800 Subject: [PATCH 128/296] test --- .azure-pipelines/code-scan.yml | 68 +++++++++---------- .../scripts/codeScan/copyright/copy_right.sh | 2 - 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 3cc877f1b99..8698fdb49fc 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -128,39 +128,39 @@ stages: docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" displayName: 'pyspelling code scan' - - stage: copyright - displayName: copyright code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - stage: copyright + # displayName: copyright code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) - # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - displayName: 'copyright code scan' \ No newline at end of file + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' + + # - script: | + # docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) + # # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + # displayName: 'copyright code scan' \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 33355341b68..41344beadb9 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -2,8 +2,6 @@ # $2: $(TARGET_BRANCH) - $(System.PullRequest.TargetBranch) # $3: $(VAL_PATH) - pip install -r requirements.txt - supported_extensions=(py, sh, yaml) From 25080a2259f233c7c3c9a1b6a80de68c743b5e0b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 15:57:17 +0800 Subject: [PATCH 129/296] test --- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index e4595a69e61..9017056fbea 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -2,8 +2,10 @@ # $2: $(BUILD_DIRECTORY) - $(Build.SourcesDirectory) pip install pyspelling + pip install aspell pip install aspell-en + pip install -r requirements.txt From a6433e972b0d925ba0a3470106c83f78722f31d9 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:02:50 +0800 Subject: [PATCH 130/296] test --- .azure-pipelines/code-scan.yml | 6 +++--- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 8698fdb49fc..d556dc1aea0 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -37,7 +37,7 @@ stages: - script: | if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -74,7 +74,7 @@ stages: - script: | if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -110,7 +110,7 @@ stages: - script: | if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 9017056fbea..e4595a69e61 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -2,10 +2,8 @@ # $2: $(BUILD_DIRECTORY) - $(Build.SourcesDirectory) pip install pyspelling - pip install aspell pip install aspell-en - pip install -r requirements.txt From 65d1234402ddb9b7cbd74d2bbc87d2815df781d6 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:03:46 +0800 Subject: [PATCH 131/296] test --- .azure-pipelines/code-scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d556dc1aea0..45ccf92d206 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -36,7 +36,7 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} @@ -73,7 +73,7 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} @@ -109,7 +109,7 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} From 5cdcae6939628905791d2e1779987f49d77becd6 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:21:18 +0800 Subject: [PATCH 132/296] test --- .azure-pipelines/code-scan.yml | 14 +++++++------- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 2 +- .azure-pipelines/scripts/codeScan/pylint/pylint.sh | 10 ++++++++++ .../scripts/codeScan/pyspelling/pyspelling.sh | 5 +++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 45ccf92d206..c468ba70693 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -36,8 +36,8 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -52,7 +52,7 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH) $(target_path)" + docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" displayName: 'bandit code scan' @@ -73,8 +73,8 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -109,8 +109,8 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 88159ddb8c5..4c0b5f7fb45 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -4,7 +4,7 @@ pip install bandit -python -m bandit -r -lll -iii $2 > /lpot-bandit.log +python -m bandit -r -lll -iii $1 > $1/lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 1afd7f7e975..5d1ae712d52 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -2,7 +2,17 @@ # $2: $(target_path) pip install pylint==2.12.1 +pip install google +pip install horovod +pip install ofa +pip install fvcore +pip install autograd +pip install pymoo +pip install tf_slim +pip install transformers pip install -r requirements.txt +pip install onnxruntime_extensions + python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index e4595a69e61..6df1d1a058c 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -2,10 +2,11 @@ # $2: $(BUILD_DIRECTORY) - $(Build.SourcesDirectory) pip install pyspelling -pip install aspell -pip install aspell-en pip install -r requirements.txt +apt-get install aspell +apt-get install aspell-en + sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling/pyspelling_conf.yaml sed -i "s|\${LPOT_REPO}|.|g" $1/pyspelling/pyspelling_conf.yaml From d1bafbcd7d63ae61c6a1e6c2fae8f9e7801459b0 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:23:35 +0800 Subject: [PATCH 133/296] test --- .azure-pipelines/code-scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index c468ba70693..b493494a0f8 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -36,7 +36,7 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} @@ -73,7 +73,7 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} @@ -109,7 +109,7 @@ stages: displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} From 684c3b17365d02914428f4a2d179da5673308cce Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:28:21 +0800 Subject: [PATCH 134/296] test --- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 6df1d1a058c..9774c9ae6fa 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -4,8 +4,8 @@ pip install pyspelling pip install -r requirements.txt -apt-get install aspell -apt-get install aspell-en +apt-get install aspell -y +apt-get install aspell-en -y sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling/pyspelling_conf.yaml From 7ba41705794eb99756c18e16141e47748d53fecc Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:33:17 +0800 Subject: [PATCH 135/296] test --- .azure-pipelines/scripts/codeScan/pylint/pylint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 5d1ae712d52..d169be1b468 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -2,6 +2,7 @@ # $2: $(target_path) pip install pylint==2.12.1 +pip install flask==2.1.3 pip install google pip install horovod pip install ofa From 85750648bfc071185ba487bc53b46dc774badbcb Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:37:31 +0800 Subject: [PATCH 136/296] test --- .azure-pipelines/code-scan.yml | 68 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index b493494a0f8..99f4523dc0d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -128,39 +128,39 @@ stages: docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" displayName: 'pyspelling code scan' - # - stage: copyright - # displayName: copyright code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # echo y | docker system prune - # displayName: 'Clean workspace' + - stage: copyright + displayName: copyright code scan + dependsOn: [] + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + displayName: 'Clean workspace' - # - checkout: self - # displayName: "Checkout out Repo" + - checkout: self + displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - - # - script: | - # docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) - # # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - # displayName: 'copyright code scan' \ No newline at end of file + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) + # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + displayName: 'copyright code scan' \ No newline at end of file From 7f77735d5ec223d103062886618d668451a5dd94 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:49:27 +0800 Subject: [PATCH 137/296] test --- .azure-pipelines/code-scan.yml | 208 ++++++++++++++++----------------- 1 file changed, 104 insertions(+), 104 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 99f4523dc0d..097e061c105 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -18,115 +18,115 @@ variables: TARGET_BRANCH: $(System.PullRequest.TargetBranch) stages: - - stage: bandit - displayName: bandit code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # sudo docker system prune || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - stage: bandit + # displayName: bandit code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # sudo docker system prune || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - displayName: 'bandit code scan' - - - - stage: pylint - displayName: pylint code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' + + # - script: | + # docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" + # displayName: 'bandit code scan' + + + # - stage: pylint + # displayName: pylint code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" - displayName: 'pylint code scan' - - - stage: pyspelling - displayName: pyspelling code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - displayName: 'Clean workspace' + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' + + # - script: | + # docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" + # displayName: 'pylint code scan' + + # - stage: pyspelling + # displayName: pyspelling code scan + # dependsOn: [] + # jobs: + # - job: Build + # displayName: Build + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # # echo y | docker system prune + # displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + # - checkout: self + # displayName: "Checkout out Repo" - - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" - displayName: 'pyspelling code scan' + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: 'Clean Docker' + + # - script: | + # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" + # displayName: 'pyspelling code scan' - stage: copyright displayName: copyright code scan @@ -161,6 +161,6 @@ stages: displayName: 'Clean Docker' - script: | - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH) + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" displayName: 'copyright code scan' \ No newline at end of file From 26f4574d85669842bea6dec3efbaa0c355efa9bd Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:51:04 +0800 Subject: [PATCH 138/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 097e061c105..bed546b41a1 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -161,6 +161,6 @@ stages: displayName: 'Clean Docker' - script: | - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" displayName: 'copyright code scan' \ No newline at end of file From 225a7da3620c46aaecc50acf151405fe5d2b9d36 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 16:51:59 +0800 Subject: [PATCH 139/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index bed546b41a1..94134489961 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -161,6 +161,6 @@ stages: displayName: 'Clean Docker' - script: | - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "pwd && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" displayName: 'copyright code scan' \ No newline at end of file From c9f2025fcb5c0633073cb976b12d7b064a9f4de4 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Wed, 10 Aug 2022 16:54:36 +0800 Subject: [PATCH 140/296] Suyue/ut update (#16) --- .azure-pipelines/scripts/install_nc.sh | 2 +- .azure-pipelines/scripts/install_nc_full.sh | 2 +- .azure-pipelines/ut-basic.yml | 2 +- .azure-pipelines/ut-ux.yml | 2 +- neural_coder/__init__.py | 1 - neural_compressor/ux/__init__.py | 2 +- neural_compressor/version.py | 1 - 7 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/scripts/install_nc.sh b/.azure-pipelines/scripts/install_nc.sh index 5b6fdc6460d..38fd439404b 100644 --- a/.azure-pipelines/scripts/install_nc.sh +++ b/.azure-pipelines/scripts/install_nc.sh @@ -3,5 +3,5 @@ cd /neural-compressor python -m pip install --no-cache-dir -r requirements.txt python setup.py sdist bdist_wheel -pip install dist/neural_compressor-*.whl +pip install dist/neural_compressor*.whl pip list \ No newline at end of file diff --git a/.azure-pipelines/scripts/install_nc_full.sh b/.azure-pipelines/scripts/install_nc_full.sh index b99d2ab305f..b3f59626dfd 100644 --- a/.azure-pipelines/scripts/install_nc_full.sh +++ b/.azure-pipelines/scripts/install_nc_full.sh @@ -3,5 +3,5 @@ cd /neural-compressor python -m pip install --no-cache-dir -r requirements.txt python setup.py --full sdist bdist_wheel -pip install dist/neural_compressor-*.whl +pip install dist/neural_compressor*.whl pip list \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index fecf3fb59d2..4e7ec62d2bc 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -8,7 +8,7 @@ pr: include: - neural_compressor exclude: - - neural_compressor.ux + - neural_compressor/ux pool: suyue-test diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index dd35dde3896..41922165d84 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -6,7 +6,7 @@ pr: - master paths: include: - - neural_compressor.ux + - neural_compressor/ux pool: suyue-test diff --git a/neural_coder/__init__.py b/neural_coder/__init__.py index 304699fab90..46dd8eee68c 100644 --- a/neural_coder/__init__.py +++ b/neural_coder/__init__.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# for test only from .interface import enable from .interface import bench diff --git a/neural_compressor/ux/__init__.py b/neural_compressor/ux/__init__.py index faabe895b72..9100b4bb131 100644 --- a/neural_compressor/ux/__init__.py +++ b/neural_compressor/ux/__init__.py @@ -12,5 +12,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# for test only + """The app package contains all components required for UX server.""" diff --git a/neural_compressor/version.py b/neural_compressor/version.py index d9478f11caa..aa09939a643 100644 --- a/neural_compressor/version.py +++ b/neural_compressor/version.py @@ -14,6 +14,5 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# for test __version__ = "1.13" From ca415ed73fe7258ef748c22ff9601d43f8684b14 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:13:00 +0800 Subject: [PATCH 141/296] test --- .azure-pipelines/code-scan.yml | 116 ++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 94134489961..5f6c703ab5f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -12,14 +12,60 @@ pool: suyue-test variables: IMAGE_NAME: 'code-scan' IMAGE_TAG: '1.0' - VAL_PATH: /neural_compressor/.azure-pipelines/scripts/codeScan - target_path: "neural_compressor" + VAL_PATH: '/neural_compressor/.azure-pipelines/scripts/codeScan' + TARGET_PATH: '/neural_compressor' BUILD_DIRECTORY: $(Build.SourcesDirectory) TARGET_BRANCH: $(System.PullRequest.TargetBranch) + BANDIT_CONTAINER_NAME: 'bandit' stages: - # - stage: bandit - # displayName: bandit code scan + - stage: bandit + displayName: bandit code scan + dependsOn: [] + jobs: + - job: Build + displayName: Build + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # sudo docker system prune || true + # echo y | docker system prune + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + + - script: | + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" + docker cp $(BANDIT_CONTAINER_NAME): $1/lpot-bandit.log $1/lpot-bandit.log + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $1/lpot-bandit.log + artifact: $(BANDIT_CONTAINER_NAME) + publishLocation: 'pipeline' + displayName: 'bandit code scan' + + + # - stage: pylint + # displayName: pylint code scan # dependsOn: [] # jobs: # - job: Build @@ -28,7 +74,6 @@ stages: # - script: | # echo ${BUILD_SOURCESDIRECTORY} # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # sudo docker system prune || true # # echo y | docker system prune # displayName: 'Clean workspace' @@ -52,12 +97,11 @@ stages: # displayName: 'Clean Docker' # - script: | - # docker run --disable-content-trust --privileged --name="bandit" --hostname="bandit-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - # displayName: 'bandit code scan' - + # docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" + # displayName: 'pylint code scan' - # - stage: pylint - # displayName: pylint code scan + # - stage: pyspelling + # displayName: pyspelling code scan # dependsOn: [] # jobs: # - job: Build @@ -89,11 +133,11 @@ stages: # displayName: 'Clean Docker' # - script: | - # docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" - # displayName: 'pylint code scan' - - # - stage: pyspelling - # displayName: pyspelling code scan + # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" + # displayName: 'pyspelling code scan' + + # - stage: copyright + # displayName: copyright code scan # dependsOn: [] # jobs: # - job: Build @@ -125,42 +169,6 @@ stages: # displayName: 'Clean Docker' # - script: | - # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" - # displayName: 'pyspelling code scan' - - - stage: copyright - displayName: copyright code scan - dependsOn: [] - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "pwd && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - displayName: 'copyright code scan' \ No newline at end of file + # docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "pwd && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + # # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + # displayName: 'copyright code scan' \ No newline at end of file From c9e18ede5e519bf570fe02cb14ed23307137e239 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:13:15 +0800 Subject: [PATCH 142/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 5f6c703ab5f..226542718c0 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -54,7 +54,7 @@ stages: - script: | docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - docker cp $(BANDIT_CONTAINER_NAME): $1/lpot-bandit.log $1/lpot-bandit.log + docker cp $(BANDIT_CONTAINER_NAME):$1/lpot-bandit.log $1/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: From 2f812bf0c81a11129f9ab510efd82578c544260d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:17:32 +0800 Subject: [PATCH 143/296] test --- .azure-pipelines/code-scan.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 226542718c0..3ed1abbebab 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -53,8 +53,9 @@ stages: displayName: 'Clean Docker' - script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - docker cp $(BANDIT_CONTAINER_NAME):$1/lpot-bandit.log $1/lpot-bandit.log + # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" + docker cp "bandit":$1/lpot-bandit.log $1/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: From a5026896a50f05291cc9c730283922a8aed251d9 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:21:59 +0800 Subject: [PATCH 144/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 3ed1abbebab..919840627d4 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -55,11 +55,11 @@ stages: - script: | # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - docker cp "bandit":$1/lpot-bandit.log $1/lpot-bandit.log + docker cp "bandit":$(VAL_PATH)/lpot-bandit.log $(VAL_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: $1/lpot-bandit.log + targetPath: $(VAL_PATH)/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: 'pipeline' displayName: 'bandit code scan' From a55b04e4995a8b4d533a62631937fb21229e9257 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:24:20 +0800 Subject: [PATCH 145/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 919840627d4..9ffd645502d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -55,11 +55,11 @@ stages: - script: | # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - docker cp "bandit":$(VAL_PATH)/lpot-bandit.log $(VAL_PATH)/lpot-bandit.log + docker cp "bandit":$(VAL_PATH)/lpot-bandit.log /lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: $(VAL_PATH)/lpot-bandit.log + targetPath: /lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: 'pipeline' displayName: 'bandit code scan' From 9eecf6f5d8b84b2324ed9184126ed69322607f17 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:25:40 +0800 Subject: [PATCH 146/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9ffd645502d..bbcc110174d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -55,11 +55,11 @@ stages: - script: | # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - docker cp "bandit":$(VAL_PATH)/lpot-bandit.log /lpot-bandit.log + docker cp "bandit":$(VAL_PATH)/lpot-bandit.log /neural_compressor/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: /lpot-bandit.log + targetPath: /neural_compressor/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: 'pipeline' displayName: 'bandit code scan' From 114b5ccb46023f99b82160d72d6cb2bc16026ce6 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Wed, 10 Aug 2022 17:27:20 +0800 Subject: [PATCH 147/296] test --- .azure-pipelines/code-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index bbcc110174d..b4d2dd0deda 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -55,6 +55,7 @@ stages: - script: | # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" + pwd docker cp "bandit":$(VAL_PATH)/lpot-bandit.log /neural_compressor/lpot-bandit.log - task: PublishPipelineArtifact@1 From a76aaf195071a50c777e924f74a74bcc08398a3d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 09:39:05 +0800 Subject: [PATCH 148/296] test --- .azure-pipelines/code-scan.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index b4d2dd0deda..335f12dfc96 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -17,6 +17,7 @@ variables: BUILD_DIRECTORY: $(Build.SourcesDirectory) TARGET_BRANCH: $(System.PullRequest.TargetBranch) BANDIT_CONTAINER_NAME: 'bandit' + CURRENT_PATH: $(pwd) stages: - stage: bandit @@ -56,11 +57,11 @@ stages: # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" pwd - docker cp "bandit":$(VAL_PATH)/lpot-bandit.log /neural_compressor/lpot-bandit.log + docker cp "bandit":$(VAL_PATH)/lpot-bandit.log $(CURRENT_PATH)/$(VAL_PATH)/bandit/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: /neural_compressor/lpot-bandit.log + targetPath: $(CURRENT_PATH)/$(VAL_PATH)/bandit/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: 'pipeline' displayName: 'bandit code scan' From 7fa4d1784ed143625d083a31ea80603ead163920 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 09:41:27 +0800 Subject: [PATCH 149/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 335f12dfc96..e8e3706a004 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -7,7 +7,7 @@ pr: # pr: none # # - master -pool: suyue-test +pool: docker-agent variables: IMAGE_NAME: 'code-scan' From b5d73fa6d9dbd0c86a9045ba622f4d230891f86c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:05:00 +0800 Subject: [PATCH 150/296] test --- .azure-pipelines/code-scan.yml | 127 ++++++++---------- .../scripts/codeScan/bandit/bandit.sh | 5 +- 2 files changed, 59 insertions(+), 73 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index e8e3706a004..352e279b3af 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,71 +1,60 @@ -trigger: +trigger: - code-scan - -pr: - - master - -# pr: none -# # - master - +pr: none pool: docker-agent variables: - IMAGE_NAME: 'code-scan' - IMAGE_TAG: '1.0' - VAL_PATH: '/neural_compressor/.azure-pipelines/scripts/codeScan' - TARGET_PATH: '/neural_compressor' - BUILD_DIRECTORY: $(Build.SourcesDirectory) + IMAGE_NAME: "code-scan" + IMAGE_TAG: "1.0" + CODE_SCAN_PATH: "/neural_compressor/.azure-pipelines/scripts/codeScan" + CODE_SCAN_LOG_PATH: "/neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" + TARGET_PATH: "/neural_compressor" TARGET_BRANCH: $(System.PullRequest.TargetBranch) - BANDIT_CONTAINER_NAME: 'bandit' - CURRENT_PATH: $(pwd) + BANDIT_CONTAINER_NAME: "bandit" + CURRENT_PATH: $(Build.SourcesDirectory) stages: - - stage: bandit - displayName: bandit code scan + - stage: BanditCodeScan + displayName: Bandit Code Scan dependsOn: [] jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # sudo docker system prune || true - # echo y | docker system prune - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(VAL_PATH)/bandit/bandit.sh $(VAL_PATH)" - pwd - docker cp "bandit":$(VAL_PATH)/lpot-bandit.log $(CURRENT_PATH)/$(VAL_PATH)/bandit/lpot-bandit.log - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(VAL_PATH)/bandit/lpot-bandit.log - artifact: $(BANDIT_CONTAINER_NAME) - publishLocation: 'pipeline' - displayName: 'bandit code scan' - + - job: Bandit + displayName: Bandit + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh --$(TARGET_PATH)" + docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: (CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + artifact: $(BANDIT_CONTAINER_NAME) + publishLocation: "pipeline" # - stage: pylint # displayName: pylint code scan @@ -79,10 +68,10 @@ stages: # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true # # echo y | docker system prune # displayName: 'Clean workspace' - + # - checkout: self # displayName: "Checkout out Repo" - + # - script: | # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . @@ -115,10 +104,10 @@ stages: # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true # # echo y | docker system prune # displayName: 'Clean workspace' - + # - checkout: self # displayName: "Checkout out Repo" - + # - script: | # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . @@ -136,9 +125,9 @@ stages: # displayName: 'Clean Docker' # - script: | - # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(BUILD_DIRECTORY)" + # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(CURRENT_PATH)" # displayName: 'pyspelling code scan' - + # - stage: copyright # displayName: copyright code scan # dependsOn: [] @@ -151,10 +140,10 @@ stages: # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true # # echo y | docker system prune # displayName: 'Clean workspace' - + # - checkout: self # displayName: "Checkout out Repo" - + # - script: | # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . @@ -172,6 +161,6 @@ stages: # displayName: 'Clean Docker' # - script: | - # docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "pwd && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" + # docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "pwd && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" # # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - # displayName: 'copyright code scan' \ No newline at end of file + # displayName: 'copyright code scan' diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 4c0b5f7fb45..c23898d7f5b 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,10 +1,7 @@ -# $1: $(VAL_PATH) -# $2: $(target_path) - pip install bandit -python -m bandit -r -lll -iii $1 > $1/lpot-bandit.log +python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then From c699b44b97b8e8a693a958e306761db9e7b72f1b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:07:49 +0800 Subject: [PATCH 151/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 352e279b3af..5ce7e6ba719 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,7 +1,7 @@ trigger: - code-scan pr: none -pool: docker-agent +pool: suyue-test variables: IMAGE_NAME: "code-scan" From 490fa177168f4ee2b81ba0aa2891384ae5420f8e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:11:32 +0800 Subject: [PATCH 152/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 5ce7e6ba719..c448066abfb 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -29,8 +29,8 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache. fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then From 4fa133a33e9ed99ebf700d2f2e35029ed7c824ae Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:21:12 +0800 Subject: [PATCH 153/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index c448066abfb..240491a1728 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,7 +47,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh --$(TARGET_PATH)" + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh --TARGET_PATH" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 From 170023b2cc70e3e88e80febd7c564e9a617d4c40 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:23:01 +0800 Subject: [PATCH 154/296] test --- .azure-pipelines/code-scan.yml | 2 +- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 240491a1728..8f83dbe116b 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,7 +47,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh --TARGET_PATH" + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index c23898d7f5b..d751da88e61 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,7 +1,8 @@ pip install bandit -python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log +python -m bandit -r -lll -iii $1 > /lpot-bandit.log +# python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 103416229f812012f9b10c6f2efb71cd34ddb93e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:35:20 +0800 Subject: [PATCH 155/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index d751da88e61..17e80b78430 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,7 +1,7 @@ pip install bandit -python -m bandit -r -lll -iii $1 > /lpot-bandit.log +python -m bandit -r -lll -iii > /lpot-bandit.log # python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log exit_code=$? From c54720548ec1244e5c3e715f449dee568a3915a1 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:40:47 +0800 Subject: [PATCH 156/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 17e80b78430..96237e1fd0c 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,7 +1,12 @@ pip install bandit +echo "-------------------" +pwd +echo "-------------------" +ls +echo "-------------------" -python -m bandit -r -lll -iii > /lpot-bandit.log +python -m bandit -r -lll -iii /neural_compressor > /lpot-bandit.log # python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log exit_code=$? From 5e8475f185bf50aeb67cf73bb6f347ead1fa010a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 11:44:14 +0800 Subject: [PATCH 157/296] test --- .azure-pipelines/code-scan.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 8f83dbe116b..935007c01c3 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,6 +47,15 @@ stages: inputs: targetType: "inline" script: | + echo "--------------------" + $(CURRENT_PATH) + cd $(CURRENT_PATH) + ls + echo "--------------------" + $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + ls + echo "--------------------" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log From 931565424c2362bf87e89dcfa4042e0548af6455 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 12:53:13 +0800 Subject: [PATCH 158/296] test --- .azure-pipelines/code-scan.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 935007c01c3..b121baf1bf0 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -6,8 +6,8 @@ pool: suyue-test variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" - CODE_SCAN_PATH: "/neural_compressor/.azure-pipelines/scripts/codeScan" - CODE_SCAN_LOG_PATH: "/neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" + CODE_SCAN_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan" + CODE_SCAN_LOG_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" TARGET_PATH: "/neural_compressor" TARGET_BRANCH: $(System.PullRequest.TargetBranch) BANDIT_CONTAINER_NAME: "bandit" @@ -29,8 +29,8 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache. + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -49,14 +49,18 @@ stages: script: | echo "--------------------" $(CURRENT_PATH) + echo "---------1-----------" cd $(CURRENT_PATH) + echo "---------2-----------" ls echo "--------------------" $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + echo "---------1-----------" cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + echo "---------2-----------" ls echo "--------------------" - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 From e1667329795cdd0637dc53d88527b07a0643f93f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 12:56:07 +0800 Subject: [PATCH 159/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index b121baf1bf0..3b5132633df 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -29,8 +29,8 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache. fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then From 55d7b9ebf1258440480d34df73fc3201ef44b0f8 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Thu, 11 Aug 2022 12:58:43 +0800 Subject: [PATCH 160/296] Update code-scan.yml for Azure Pipelines --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 3b5132633df..eaa55afc8a1 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -61,11 +61,11 @@ stages: ls echo "--------------------" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" - docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: (CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + targetPath: (CURRENT_PATH) artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" From 6d6ae6d814778ef165078432a157b8cd5d937f6d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 13:11:49 +0800 Subject: [PATCH 161/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 3b5132633df..ab0235a74d4 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -54,9 +54,9 @@ stages: echo "---------2-----------" ls echo "--------------------" - $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + $(CURRENT_PATH)/$(CODE_SCAN_PATH) echo "---------1-----------" - cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + cd $(CURRENT_PATH)/$(CODE_SCAN_PATH) echo "---------2-----------" ls echo "--------------------" From 80fa329565e5d3bc4e585d2c9c873b86a75b5904 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 13:15:23 +0800 Subject: [PATCH 162/296] test --- .azure-pipelines/code-scan.yml | 2 +- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index a492dae283c..a6257987335 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -65,7 +65,7 @@ stages: - task: PublishPipelineArtifact@1 inputs: - targetPath: (CURRENT_PATH) + targetPath: $(CURRENT_PATH) artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 96237e1fd0c..6c5facb4a1b 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,11 +1,5 @@ pip install bandit -echo "-------------------" -pwd -echo "-------------------" -ls -echo "-------------------" - python -m bandit -r -lll -iii /neural_compressor > /lpot-bandit.log # python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log From 007bf7379315f4a96537b923790b770aa7e3364b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 13:20:16 +0800 Subject: [PATCH 163/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index a6257987335..d856fbdb756 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -65,7 +65,7 @@ stages: - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH) + targetPath: $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" From c5372167bb959b48fd38af7325b641ca293c4908 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 13:25:09 +0800 Subject: [PATCH 164/296] test --- .azure-pipelines/code-scan.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d856fbdb756..dc35c88dba9 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -30,7 +30,7 @@ stages: displayName: "Checkout out Repo" - script: | if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache. + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -47,25 +47,12 @@ stages: inputs: targetType: "inline" script: | - echo "--------------------" - $(CURRENT_PATH) - echo "---------1-----------" - cd $(CURRENT_PATH) - echo "---------2-----------" - ls - echo "--------------------" - $(CURRENT_PATH)/$(CODE_SCAN_PATH) - echo "---------1-----------" - cd $(CURRENT_PATH)/$(CODE_SCAN_PATH) - echo "---------2-----------" - ls - echo "--------------------" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan + targetPath: artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" From 177004bbe3f45fd9e5731d2b89c4b261b7d7d0e4 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 13:38:54 +0800 Subject: [PATCH 165/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index dc35c88dba9..200c6eb7031 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,12 +47,12 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" + docker run --rm --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: + targetPath: $(CURRENT_PATH) artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" From b879d4c07db9d295ed685919b19206b6e765a983 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 13:54:37 +0800 Subject: [PATCH 166/296] test --- .azure-pipelines/code-scan.yml | 8 ++++---- .azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log | 1 + .gitignore | 1 - 3 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 .azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 200c6eb7031..0d66b4a8d58 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -30,7 +30,7 @@ stages: displayName: "Checkout out Repo" - script: | if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} --no-cache . + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -47,12 +47,12 @@ stages: inputs: targetType: "inline" script: | - docker run --rm --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" - docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/lpot-bandit.log + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" + docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH) + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" diff --git a/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log b/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log new file mode 100644 index 00000000000..9d07aa0df55 --- /dev/null +++ b/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log @@ -0,0 +1 @@ +111 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6f4be8433c0..b222b2390cc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ *.csv *.pb *.ckpt -*.log *.swp *.onnx *.so From d56cda01bc4cc0a0c66132c407e08a397692c0f9 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:00:29 +0800 Subject: [PATCH 167/296] test --- .azure-pipelines/code-scan.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 0d66b4a8d58..74d25843851 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,6 +47,15 @@ stages: inputs: targetType: "inline" script: | + echo "------------------------" + cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + ls -a + echo "------------------------" + cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan + ls -a + echo "------------------------" + + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log From a073dfa9aaab1fcb6d943b801aa4d8dc585b5b88 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:06:55 +0800 Subject: [PATCH 168/296] test --- .azure-pipelines/code-scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 74d25843851..dcdfa290139 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,13 +47,13 @@ stages: inputs: targetType: "inline" script: | - echo "------------------------" - cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) - ls -a echo "------------------------" cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan ls -a echo "------------------------" +- cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan/scanLog + ls -a + echo "------------------------" docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" From fa138a31a90d10299bb8b89903c5ef5abd39f03a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:14:16 +0800 Subject: [PATCH 169/296] test --- .azure-pipelines/code-scan.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index dcdfa290139..7b4d3c76212 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -6,8 +6,8 @@ pool: suyue-test variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" - CODE_SCAN_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan" - CODE_SCAN_LOG_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" + CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" + CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" TARGET_PATH: "/neural_compressor" TARGET_BRANCH: $(System.PullRequest.TargetBranch) BANDIT_CONTAINER_NAME: "bandit" @@ -47,15 +47,6 @@ stages: inputs: targetType: "inline" script: | - echo "------------------------" - cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan - ls -a - echo "------------------------" -- cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan/scanLog - ls -a - echo "------------------------" - - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log From 116423f9dbfdf4494edcfdac6c7fff0222dfc2d2 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:17:26 +0800 Subject: [PATCH 170/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 7b4d3c76212..422a6e179d1 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -47,7 +47,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash /$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 From fe1af55a1a356962dd2f8b5b69623cc3465ff63b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:18:44 +0800 Subject: [PATCH 171/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 422a6e179d1..8cb5943b191 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -29,7 +29,7 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ $(docker images | grep -i ${IMAGE_NAME}) ]]; then + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} From 0ee8c644c7acb3137e771e8860d3733460f2ccc8 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:25:53 +0800 Subject: [PATCH 172/296] test --- .azure-pipelines/code-scan.yml | 8 ++++---- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 8cb5943b191..1366b2c8ece 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -7,7 +7,7 @@ variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" - CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" + CODE_SCAN_LOG_PATH: "scanLog" TARGET_PATH: "/neural_compressor" TARGET_BRANCH: $(System.PullRequest.TargetBranch) BANDIT_CONTAINER_NAME: "bandit" @@ -47,12 +47,12 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/bandit/bandit.sh $(TARGET_PATH)" - docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" + docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 6c5facb4a1b..c343b9f10a3 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,7 +1,6 @@ pip install bandit -python -m bandit -r -lll -iii /neural_compressor > /lpot-bandit.log -# python -m bandit -r -lll -iii $(TARGET_PATH) > /lpot-bandit.log +python -m bandit -r -lll -iii ${TARGET_PATH} > /lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 17465d259b026d19c1ceee1a2e2409b21614d809 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:37:44 +0800 Subject: [PATCH 173/296] test --- .azure-pipelines/code-scan.yml | 83 +++++++++++-------- .../scripts/codeScan/pylint/pylint.sh | 2 +- 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 1366b2c8ece..1dd0946e316 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,8 +9,12 @@ variables: CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: "scanLog" TARGET_PATH: "/neural_compressor" - TARGET_BRANCH: $(System.PullRequest.TargetBranch) BANDIT_CONTAINER_NAME: "bandit" + PYLINT_CONTAINER_NAME: "pylint" + PYSPELLING_CONTAINER_NAME: "pyspelling" + COPYRIGHT_CONTAINER_NAME: "copyright" + + TARGET_BRANCH: $(System.PullRequest.TargetBranch) CURRENT_PATH: $(Build.SourcesDirectory) stages: @@ -47,8 +51,8 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" - docker cp 'bandit':/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" + docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: @@ -56,41 +60,52 @@ stages: artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" - # - stage: pylint - # displayName: pylint code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # echo y | docker system prune - # displayName: 'Clean workspace' + - stage: PylintCodeScan + displayName: Pylint Code Scan + dependsOn: [] + jobs: + - job: Pylint + displayName: Pylint + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" - # - checkout: self - # displayName: "Checkout out Repo" + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh --TARGET_PATH $(TARGET_PATH)" + docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + artifact: $(PYLINT_CONTAINER_NAME) + publishLocation: "pipeline" + - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - # - script: | - # docker run --disable-content-trust --privileged --name="pylint" --hostname="pylint-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pylint/pylint.sh $(VAL_PATH) $(target_path)" - # displayName: 'pylint code scan' + - script: | + displayName: 'pylint code scan' # - stage: pyspelling # displayName: pyspelling code scan diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index d169be1b468..cc0830f8872 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -16,7 +16,7 @@ pip install onnxruntime_extensions -python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./$2 +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ${TARGET_PATH} > /lpot-pylint.json exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 356ce8019b6868f1dc385a1743c8e8bdfd417e14 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:38:44 +0800 Subject: [PATCH 174/296] test --- .azure-pipelines/code-scan.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 1dd0946e316..f0c224d485f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -101,11 +101,6 @@ stages: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json artifact: $(PYLINT_CONTAINER_NAME) publishLocation: "pipeline" - - - - - script: | - displayName: 'pylint code scan' # - stage: pyspelling # displayName: pyspelling code scan From 355203958dbf8395248f9cc4e0d8ea9cb1e1ab7c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 14:54:39 +0800 Subject: [PATCH 175/296] test --- .azure-pipelines/code-scan.yml | 42 +++++++++++++++++++ .../scripts/codeScan/pyspelling/pyspelling.sh | 11 ++--- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f0c224d485f..3ab56ffe336 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -102,6 +102,48 @@ stages: artifact: $(PYLINT_CONTAINER_NAME) publishLocation: "pipeline" + - stage: PyspellingCodeScan + displayName: Pyspelling Code Scan + dependsOn: [] + jobs: + - job: Pyspelling + displayName: Pyspelling + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --TARGET_PATH $(TARGET_PATH)/$(CODE_SCAN_PATH) --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" + docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + artifact: $(PYSPELLING_CONTAINER_NAME) + publishLocation: "pipeline" + # - stage: pyspelling # displayName: pyspelling code scan # dependsOn: [] diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 9774c9ae6fa..b6ab4a50b68 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -1,6 +1,3 @@ -# $1: $(VAL_PATH) -# $2: $(BUILD_DIRECTORY) - $(Build.SourcesDirectory) - pip install pyspelling pip install -r requirements.txt @@ -8,11 +5,11 @@ apt-get install aspell -y apt-get install aspell-en -y -sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" $1/pyspelling/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml echo "Modified config:" -cat $1/pyspelling/pyspelling_conf.yaml -pyspelling -c $1/pyspelling/pyspelling_conf.yaml > $1/pyspelling/pyspelling_output.log +cat ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml +pyspelling -c ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From 3ee9be71975e979dfc669cd29d17f32c007bd446 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:01:45 +0800 Subject: [PATCH 176/296] test --- .azure-pipelines/code-scan.yml | 206 ++++++++---------- .../scripts/codeScan/pyspelling/pyspelling.sh | 8 +- 2 files changed, 90 insertions(+), 124 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 3ab56ffe336..a4c3640ec9d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -6,9 +6,11 @@ pool: suyue-test variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" + + TARGET_PATH: "/neural_compressor" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: "scanLog" - TARGET_PATH: "/neural_compressor" + BANDIT_CONTAINER_NAME: "bandit" PYLINT_CONTAINER_NAME: "pylint" PYSPELLING_CONTAINER_NAME: "pyspelling" @@ -18,89 +20,89 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - - stage: BanditCodeScan - displayName: Bandit Code Scan - dependsOn: [] - jobs: - - job: Bandit - displayName: Bandit - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" - docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - artifact: $(BANDIT_CONTAINER_NAME) - publishLocation: "pipeline" - - - stage: PylintCodeScan - displayName: Pylint Code Scan - dependsOn: [] - jobs: - - job: Pylint - displayName: Pylint - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh --TARGET_PATH $(TARGET_PATH)" - docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - artifact: $(PYLINT_CONTAINER_NAME) - publishLocation: "pipeline" + # - stage: BanditCodeScan + # displayName: Bandit Code Scan + # dependsOn: [] + # jobs: + # - job: Bandit + # displayName: Bandit + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" + # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # artifact: $(BANDIT_CONTAINER_NAME) + # publishLocation: "pipeline" + + # - stage: PylintCodeScan + # displayName: Pylint Code Scan + # dependsOn: [] + # jobs: + # - job: Pylint + # displayName: Pylint + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh --TARGET_PATH $(TARGET_PATH)" + # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # artifact: $(PYLINT_CONTAINER_NAME) + # publishLocation: "pipeline" - stage: PyspellingCodeScan displayName: Pyspelling Code Scan @@ -144,42 +146,6 @@ stages: artifact: $(PYSPELLING_CONTAINER_NAME) publishLocation: "pipeline" - # - stage: pyspelling - # displayName: pyspelling code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # echo y | docker system prune - # displayName: 'Clean workspace' - - # - checkout: self - # displayName: "Checkout out Repo" - - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - - # - script: | - # docker run --disable-content-trust --privileged --name="pyspelling" --hostname="pyspelling-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash -x -c "cd /neural_compressor && bash $(VAL_PATH)/pyspelling/pyspelling.sh $(VAL_PATH) $(CURRENT_PATH)" - # displayName: 'pyspelling code scan' - # - stage: copyright # displayName: copyright code scan # dependsOn: [] diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index b6ab4a50b68..dba4a5c8bca 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -5,11 +5,11 @@ apt-get install aspell -y apt-get install aspell-en -y -sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml echo "Modified config:" -cat ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml -pyspelling -c ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml > /lpot_pyspelling.log +cat ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml +pyspelling -c ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From 335331d25f6a76f87393c09beae5318da95b7d2e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:06:01 +0800 Subject: [PATCH 177/296] test --- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index dba4a5c8bca..5e9078c84ef 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -4,6 +4,13 @@ pip install -r requirements.txt apt-get install aspell -y apt-get install aspell-en -y +echo "---------------------" +${TARGET_PATH} +echo "---------------------" +${PYSPELLING_CONTAINER_NAME} +echo "---------------------" +cd ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME} +echo "---------------------------------------" sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml From ad0fa292ddb6f52f8ab9c378a84f1f5e1dee4565 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:17:07 +0800 Subject: [PATCH 178/296] test --- .azure-pipelines/code-scan.yml | 2 +- .../scripts/codeScan/pyspelling/pyspelling.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index a4c3640ec9d..1e5c48fc455 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -137,7 +137,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --TARGET_PATH $(TARGET_PATH)/$(CODE_SCAN_PATH) --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 5e9078c84ef..347f79e116a 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -12,11 +12,11 @@ echo "---------------------" cd ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME} echo "---------------------------------------" -sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml echo "Modified config:" -cat ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml -pyspelling -c ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}pyspelling_conf.yaml > /lpot_pyspelling.log +cat ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml +pyspelling -c ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From a6231d5406a67eabf561e7098429ee25bbdb0684 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:17:23 +0800 Subject: [PATCH 179/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 1e5c48fc455..19d1b9f9c46 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -137,7 +137,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME) --TARGET_PATH $(TARGET_PATH)/$(CODE_SCAN_PATH)" docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - task: PublishPipelineArtifact@1 From 0bce4fa6f99aa46ca8bde6482e18fc5f5822e7d7 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:19:31 +0800 Subject: [PATCH 180/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 19d1b9f9c46..1e5c48fc455 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -137,7 +137,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME) --TARGET_PATH $(TARGET_PATH)/$(CODE_SCAN_PATH)" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - task: PublishPipelineArtifact@1 From c94b138ee96318a386589a180adc82fa700e040a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:38:11 +0800 Subject: [PATCH 181/296] test --- .azure-pipelines/code-scan.yml | 118 +++++++++--------- .../scripts/codeScan/bandit/bandit.sh | 4 + 2 files changed, 63 insertions(+), 59 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 1e5c48fc455..12d208fb805 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -20,12 +20,54 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - # - stage: BanditCodeScan - # displayName: Bandit Code Scan + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" + docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + artifact: $(BANDIT_CONTAINER_NAME) + publishLocation: "pipeline" + + # - stage: PylintCodeScan + # displayName: Pylint Code Scan # dependsOn: [] # jobs: - # - job: Bandit - # displayName: Bandit + # - job: Pylint + # displayName: Pylint # steps: # - script: | # echo ${BUILD_SOURCESDIRECTORY} @@ -53,21 +95,21 @@ stages: # inputs: # targetType: "inline" # script: | - # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" - # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh --TARGET_PATH $(TARGET_PATH)" + # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json # - task: PublishPipelineArtifact@1 # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # artifact: $(BANDIT_CONTAINER_NAME) + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # artifact: $(PYLINT_CONTAINER_NAME) # publishLocation: "pipeline" - # - stage: PylintCodeScan - # displayName: Pylint Code Scan + # - stage: PyspellingCodeScan + # displayName: Pyspelling Code Scan # dependsOn: [] # jobs: - # - job: Pylint - # displayName: Pylint + # - job: Pyspelling + # displayName: Pyspelling # steps: # - script: | # echo ${BUILD_SOURCESDIRECTORY} @@ -95,56 +137,14 @@ stages: # inputs: # targetType: "inline" # script: | - # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh --TARGET_PATH $(TARGET_PATH)" - # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" + # docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log # - task: PublishPipelineArtifact@1 # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - # artifact: $(PYLINT_CONTAINER_NAME) - # publishLocation: "pipeline" - - - stage: PyspellingCodeScan - displayName: Pyspelling Code Scan - dependsOn: [] - jobs: - - job: Pyspelling - displayName: Pyspelling - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" - docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - artifact: $(PYSPELLING_CONTAINER_NAME) - publishLocation: "pipeline" + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + # artifact: $(PYSPELLING_CONTAINER_NAME) + # publishLocation: "pipeline" # - stage: copyright # displayName: copyright code scan diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index c343b9f10a3..31f07c7e164 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,5 +1,9 @@ pip install bandit +echo "--------------------" +${TARGET_PATH} +echo "--------------------" + python -m bandit -r -lll -iii ${TARGET_PATH} > /lpot-bandit.log exit_code=$? From 0bbb2cc8bb7ab94e153debef8a621b11bed4704f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:48:15 +0800 Subject: [PATCH 182/296] test --- .azure-pipelines/code-scan.yml | 2 +- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 12d208fb805..7ffa22e7d5a 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -53,7 +53,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh --TARGET_PATH $(TARGET_PATH)" + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 31f07c7e164..f29b5eaebb5 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,10 +1,10 @@ pip install bandit echo "--------------------" -${TARGET_PATH} +$1 echo "--------------------" -python -m bandit -r -lll -iii ${TARGET_PATH} > /lpot-bandit.log +python -m bandit -r -lll -iii $1 > /lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 999cfb44c13ab2535d0d5bf54a5c574a92269aa5 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:52:39 +0800 Subject: [PATCH 183/296] test --- .azure-pipelines/code-scan.yml | 2 +- .../scripts/codeScan/pylint/pylint.sh | 2 +- .../scripts/codeScan/pyspelling/pyspelling.sh | 16 +++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 7ffa22e7d5a..9cdd0be979d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -95,7 +95,7 @@ stages: # inputs: # targetType: "inline" # script: | - # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh --TARGET_PATH $(TARGET_PATH)" + # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh $(TARGET_PATH)" # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json # - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index cc0830f8872..0b015ccb71e 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -16,7 +16,7 @@ pip install onnxruntime_extensions -python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ${TARGET_PATH} > /lpot-pylint.json +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime $1 > /lpot-pylint.json exit_code=$? if [ ${exit_code} -ne 0 ] ; then diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 347f79e116a..33176c21424 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -5,18 +5,16 @@ apt-get install aspell -y apt-get install aspell-en -y echo "---------------------" -${TARGET_PATH} +$1 echo "---------------------" -${PYSPELLING_CONTAINER_NAME} -echo "---------------------" -cd ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME} -echo "---------------------------------------" +$2 + -sed -i "s|\${VAL_REPO}|${TARGET_PATH}|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|$1|g" $1/$2/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" $1/$2/pyspelling_conf.yaml echo "Modified config:" -cat ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml -pyspelling -c ${TARGET_PATH}/${PYSPELLING_CONTAINER_NAME}/pyspelling_conf.yaml > /lpot_pyspelling.log +cat $1/$2/pyspelling_conf.yaml +pyspelling -c $1/$2/pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From 03a6a7ae1b1c0e4bafc2a2966e3a8d84879f977e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 15:52:54 +0800 Subject: [PATCH 184/296] test --- .azure-pipelines/code-scan.yml | 166 ++++++++++++++++----------------- 1 file changed, 83 insertions(+), 83 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9cdd0be979d..d0a0393b0ec 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -62,89 +62,89 @@ stages: artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" - # - stage: PylintCodeScan - # displayName: Pylint Code Scan - # dependsOn: [] - # jobs: - # - job: Pylint - # displayName: Pylint - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" - - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh $(TARGET_PATH)" - # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - # artifact: $(PYLINT_CONTAINER_NAME) - # publishLocation: "pipeline" - - # - stage: PyspellingCodeScan - # displayName: Pyspelling Code Scan - # dependsOn: [] - # jobs: - # - job: Pyspelling - # displayName: Pyspelling - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" - - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh --PYSPELLING_CONTAINER_NAME $(PYSPELLING_CONTAINER_NAME)" - # docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - # artifact: $(PYSPELLING_CONTAINER_NAME) - # publishLocation: "pipeline" + - stage: PylintCodeScan + displayName: Pylint Code Scan + dependsOn: [] + jobs: + - job: Pylint + displayName: Pylint + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh $(TARGET_PATH)" + docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + artifact: $(PYLINT_CONTAINER_NAME) + publishLocation: "pipeline" + + - stage: PyspellingCodeScan + displayName: Pyspelling Code Scan + dependsOn: [] + jobs: + - job: Pyspelling + displayName: Pyspelling + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(CURRENT_PATH)/$(CODE_SCAN_PATH) $(PYSPELLING_CONTAINER_NAME)" + docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + artifact: $(PYSPELLING_CONTAINER_NAME) + publishLocation: "pipeline" # - stage: copyright # displayName: copyright code scan From 9e1f33c8b1b1de7df408a832e217f6b1ca4d5d58 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:00:29 +0800 Subject: [PATCH 185/296] test --- .azure-pipelines/code-scan.yml | 2 +- .../scripts/codeScan/pyspelling/pyspelling.sh | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d0a0393b0ec..8ab5c071756 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -137,7 +137,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(CURRENT_PATH)/$(CODE_SCAN_PATH) $(PYSPELLING_CONTAINER_NAME)" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(TARGET_PATH)" docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 33176c21424..5c17766a911 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -4,17 +4,13 @@ pip install -r requirements.txt apt-get install aspell -y apt-get install aspell-en -y -echo "---------------------" -$1 -echo "---------------------" -$2 -sed -i "s|\${VAL_REPO}|$1|g" $1/$2/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" $1/$2/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" $1/pyspelling_conf.yaml echo "Modified config:" -cat $1/$2/pyspelling_conf.yaml -pyspelling -c $1/$2/pyspelling_conf.yaml > /lpot_pyspelling.log +cat $1/pyspelling_conf.yaml +pyspelling -c $1/pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From b92845a826ce5e6098c6f5495ae3ee41976b550c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:04:33 +0800 Subject: [PATCH 186/296] test --- .azure-pipelines/scripts/codeScan/pylint/pylint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 0b015ccb71e..a6ff9db0999 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -16,7 +16,7 @@ pip install onnxruntime_extensions -python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime $1 > /lpot-pylint.json +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime $1/$1 > /lpot-pylint.json exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 42dc109d02cebdcc4d255bae40162a695c52913c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:07:24 +0800 Subject: [PATCH 187/296] test --- .../scripts/codeScan/pyspelling/pyspelling.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 5c17766a911..9fb260d83bc 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -6,11 +6,11 @@ apt-get install aspell-en -y -sed -i "s|\${VAL_REPO}|$1|g" $1/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" $1/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|$1|g" $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml echo "Modified config:" -cat $1/pyspelling_conf.yaml -pyspelling -c $1/pyspelling_conf.yaml > /lpot_pyspelling.log +cat $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml +pyspelling -c $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From ad64b2bce1af2adec0cdeb66e31855fd13e16e50 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:10:52 +0800 Subject: [PATCH 188/296] test --- .../scripts/codeScan/pyspelling/pyspelling.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 9fb260d83bc..f03e726c73c 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -6,11 +6,11 @@ apt-get install aspell-en -y -sed -i "s|\${VAL_REPO}|$1|g" $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|$1|g" /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml echo "Modified config:" -cat $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml -pyspelling -c $1/.azure-pipelines/scripts/codeScan/pyspelling_conf.yaml > /lpot_pyspelling.log +cat /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml +pyspelling -c /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml > /lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From b609ee9b233642a0bf400f4e72e35b87226f9e1e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:17:31 +0800 Subject: [PATCH 189/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 8ab5c071756..9db945627e1 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -137,7 +137,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(TARGET_PATH)" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(TARGET_PATH)/$(CODE_SCAN_PATH)" docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - task: PublishPipelineArtifact@1 From 7c61991e6e5136b0b7b902b68d5fea89720d492f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:21:29 +0800 Subject: [PATCH 190/296] test --- .azure-pipelines/code-scan.yml | 77 ++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9db945627e1..ae77eb8a2f2 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -146,39 +146,44 @@ stages: artifact: $(PYSPELLING_CONTAINER_NAME) publishLocation: "pipeline" - # - stage: copyright - # displayName: copyright code scan - # dependsOn: [] - # jobs: - # - job: Build - # displayName: Build - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # # echo y | docker system prune - # displayName: 'Clean workspace' - - # - checkout: self - # displayName: "Checkout out Repo" - - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: 'Clean Docker' - - # - script: | - # docker run --env PYTHONPATH="/neural-compressor" ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "pwd && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - # # docker run --disable-content-trust --privileged --name="copyright" --hostname="copyright-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && bash $(VAL_PATH)/copyright/copy_right.sh $(target_path) $(TARGET_BRANCH) $(VAL_PATH)" - # displayName: 'copyright code scan' + - stage: CopyRight + displayName: CopyRight Code Scan + dependsOn: [] + jobs: + - job: CopyRight + displayName: CopyRight + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/copy_right.sh" + docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + artifact: $(PYSPELLING_CONTAINER_NAME) + publishLocation: "pipeline" \ No newline at end of file From ffab2980e91d1678a9c0082880197b3024451e61 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:24:19 +0800 Subject: [PATCH 191/296] test --- .azure-pipelines/code-scan.yml | 250 +++++++++--------- .../scripts/codeScan/copyright/copy_right.sh | 1 - 2 files changed, 125 insertions(+), 126 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index ae77eb8a2f2..2d0b6f02d28 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -20,131 +20,131 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - - stage: BanditCodeScan - displayName: Bandit Code Scan - dependsOn: [] - jobs: - - job: Bandit - displayName: Bandit - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" - docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - artifact: $(BANDIT_CONTAINER_NAME) - publishLocation: "pipeline" - - - stage: PylintCodeScan - displayName: Pylint Code Scan - dependsOn: [] - jobs: - - job: Pylint - displayName: Pylint - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh $(TARGET_PATH)" - docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - artifact: $(PYLINT_CONTAINER_NAME) - publishLocation: "pipeline" - - - stage: PyspellingCodeScan - displayName: Pyspelling Code Scan - dependsOn: [] - jobs: - - job: Pyspelling - displayName: Pyspelling - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(TARGET_PATH)/$(CODE_SCAN_PATH)" - docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - artifact: $(PYSPELLING_CONTAINER_NAME) - publishLocation: "pipeline" + # - stage: BanditCodeScan + # displayName: Bandit Code Scan + # dependsOn: [] + # jobs: + # - job: Bandit + # displayName: Bandit + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" + # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # artifact: $(BANDIT_CONTAINER_NAME) + # publishLocation: "pipeline" + + # - stage: PylintCodeScan + # displayName: Pylint Code Scan + # dependsOn: [] + # jobs: + # - job: Pylint + # displayName: Pylint + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh $(TARGET_PATH)" + # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # artifact: $(PYLINT_CONTAINER_NAME) + # publishLocation: "pipeline" + + # - stage: PyspellingCodeScan + # displayName: Pyspelling Code Scan + # dependsOn: [] + # jobs: + # - job: Pyspelling + # displayName: Pyspelling + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(TARGET_PATH)/$(CODE_SCAN_PATH)" + # docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + # artifact: $(PYSPELLING_CONTAINER_NAME) + # publishLocation: "pipeline" - stage: CopyRight displayName: CopyRight Code Scan diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 41344beadb9..bbfbec73987 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -1,7 +1,6 @@ # $1: $(target_path) # $2: $(TARGET_BRANCH) - $(System.PullRequest.TargetBranch) # $3: $(VAL_PATH) - supported_extensions=(py, sh, yaml) From 4b78d1be056243b19ad96001dd73abbf0c35be50 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:25:19 +0800 Subject: [PATCH 192/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 2d0b6f02d28..0607e3dd2c4 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -179,7 +179,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/copy_right.sh" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh" docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - task: PublishPipelineArtifact@1 From 79a116e7fe5a276c02cab9c0bf7bc7bd308cd15a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:27:09 +0800 Subject: [PATCH 193/296] test --- .azure-pipelines/code-scan.yml | 2 +- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 0607e3dd2c4..951f8a87265 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -179,7 +179,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh $(System.PullRequest.TargetBranch)" docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index bbfbec73987..183a11b9eb8 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -3,6 +3,10 @@ # $3: $(VAL_PATH) supported_extensions=(py, sh, yaml) + + echo "-----------------------" + $1 + echo "-----------------------" set -xe git config --global --add safe.directory /neural_compressor From 590caa4204f700dff2117b3eb455c4437856c85e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:29:15 +0800 Subject: [PATCH 194/296] test --- .azure-pipelines/code-scan.yml | 2 +- .azure-pipelines/scripts/codeScan/copyright/copy_right.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 951f8a87265..bb8d4c64a2f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -179,7 +179,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh $(System.PullRequest.TargetBranch)" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh $(TARGET_BRANCH)" docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - task: PublishPipelineArtifact@1 diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index 183a11b9eb8..ed69d5e0b21 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -5,7 +5,7 @@ supported_extensions=(py, sh, yaml) echo "-----------------------" - $1 + echo $1 echo "-----------------------" set -xe From 412a86ffd86bd6440d4fdf5510734e4c667d24fe Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:44:41 +0800 Subject: [PATCH 195/296] test --- .azure-pipelines/code-scan.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index bb8d4c64a2f..fda2991cfb9 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -179,7 +179,8 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh $(TARGET_BRANCH)" + $(TARGET_BRANCH) + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - task: PublishPipelineArtifact@1 From 9a176fe391033eeb5ac9d19ee5a49a3903d2ec78 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:57:47 +0800 Subject: [PATCH 196/296] test --- .azure-pipelines/code-scan.yml | 59 ++++++++++--------- .../scripts/codeScan/copyright/copy_right.sh | 4 -- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index fda2991cfb9..b3731919cd0 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,6 +1,7 @@ trigger: - code-scan -pr: none +pr: + - master pool: suyue-test variables: @@ -20,34 +21,34 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - # - stage: BanditCodeScan - # displayName: Bandit Code Scan - # dependsOn: [] - # jobs: - # - job: Bandit - # displayName: Bandit - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" # - task: Bash@3 # inputs: diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index ed69d5e0b21..bbfbec73987 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -3,10 +3,6 @@ # $3: $(VAL_PATH) supported_extensions=(py, sh, yaml) - - echo "-----------------------" - echo $1 - echo "-----------------------" set -xe git config --global --add safe.directory /neural_compressor From 7bf4c3669ca2bf54e2a9f1276ce2f960aa35ed4d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 16:59:29 +0800 Subject: [PATCH 197/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index f29b5eaebb5..aeb6c20857b 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -4,7 +4,7 @@ echo "--------------------" $1 echo "--------------------" -python -m bandit -r -lll -iii $1 > /lpot-bandit.log +python -m bandit -r -lll -iii $1/neural_compressor > /lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 718df039b7c228fc17f7f629fbae89ad6d61edce Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:06:25 +0800 Subject: [PATCH 198/296] test --- .azure-pipelines/code-scan.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index b3731919cd0..02e2d1f84eb 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -50,18 +50,18 @@ stages: docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" - # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" + docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # artifact: $(BANDIT_CONTAINER_NAME) - # publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + artifact: $(BANDIT_CONTAINER_NAME) + publishLocation: "pipeline" # - stage: PylintCodeScan # displayName: Pylint Code Scan From 5145e8eaa52eff4ed54a92e38025e6b9ad96d09c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:12:17 +0800 Subject: [PATCH 199/296] test --- .azure-pipelines/code-scan.yml | 80 +++++++++++++++++----------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 02e2d1f84eb..6ee87afc7eb 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -21,47 +21,47 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - - stage: BanditCodeScan - displayName: Bandit Code Scan - dependsOn: [] - jobs: - - job: Bandit - displayName: Bandit - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" + # - stage: BanditCodeScan + # displayName: Bandit Code Scan + # dependsOn: [] + # jobs: + # - job: Bandit + # displayName: Bandit + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" - docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" + # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - artifact: $(BANDIT_CONTAINER_NAME) - publishLocation: "pipeline" + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # artifact: $(BANDIT_CONTAINER_NAME) + # publishLocation: "pipeline" # - stage: PylintCodeScan # displayName: Pylint Code Scan @@ -180,7 +180,7 @@ stages: inputs: targetType: "inline" script: | - $(TARGET_BRANCH) + echo $(TARGET_BRANCH) docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From b17c512ee406075e49f93a9cd1993f159562f200 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Thu, 11 Aug 2022 17:16:18 +0800 Subject: [PATCH 200/296] Update code-scan.yml for Azure Pipelines --- .azure-pipelines/code-scan.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 6ee87afc7eb..b59788086cc 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -181,6 +181,8 @@ stages: targetType: "inline" script: | echo $(TARGET_BRANCH) + echo $(System.PullRequest.TargetBranch) + echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From d3eda070767fef9abd90f64a74a49a177b57a7ac Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:18:49 +0800 Subject: [PATCH 201/296] test --- .azure-pipelines/code-scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index b59788086cc..ab20d7c600a 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -180,9 +180,9 @@ stages: inputs: targetType: "inline" script: | - echo $(TARGET_BRANCH) - echo $(System.PullRequest.TargetBranch) - echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} + echo "$(TARGET_BRANCH)"" + echo "$(System.PullRequest.TargetBranch)"" + echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From 65d121dae0ed108ffcf6de99be23155b91bfcbe5 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:19:02 +0800 Subject: [PATCH 202/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index ab20d7c600a..c84f277c848 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -180,8 +180,8 @@ stages: inputs: targetType: "inline" script: | - echo "$(TARGET_BRANCH)"" - echo "$(System.PullRequest.TargetBranch)"" + echo "$(TARGET_BRANCH)" + echo "$(System.PullRequest.TargetBranch)" echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From e0a5a8b9adac1909257f8357560fa2680f661749 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:21:25 +0800 Subject: [PATCH 203/296] test --- .azure-pipelines/code-scan.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index c84f277c848..69f796fa4b7 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -151,6 +151,12 @@ stages: displayName: CopyRight Code Scan dependsOn: [] jobs: + - job: pr + steps: + - script: | + echo "$(TARGET_BRANCH)" + echo "$(System.PullRequest.TargetBranch)" + echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" - job: CopyRight displayName: CopyRight steps: @@ -180,9 +186,6 @@ stages: inputs: targetType: "inline" script: | - echo "$(TARGET_BRANCH)" - echo "$(System.PullRequest.TargetBranch)" - echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From f0791bedc9d0ba4f7db03926b94716b56692d9e8 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:27:19 +0800 Subject: [PATCH 204/296] test --- .azure-pipelines/code-scan.yml | 9 +++------ .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 69f796fa4b7..c84f277c848 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -151,12 +151,6 @@ stages: displayName: CopyRight Code Scan dependsOn: [] jobs: - - job: pr - steps: - - script: | - echo "$(TARGET_BRANCH)" - echo "$(System.PullRequest.TargetBranch)" - echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" - job: CopyRight displayName: CopyRight steps: @@ -186,6 +180,9 @@ stages: inputs: targetType: "inline" script: | + echo "$(TARGET_BRANCH)" + echo "$(System.PullRequest.TargetBranch)" + echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index aeb6c20857b..78f9ec1ce01 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -4,7 +4,7 @@ echo "--------------------" $1 echo "--------------------" -python -m bandit -r -lll -iii $1/neural_compressor > /lpot-bandit.log +python -m bandit -r -lll -iii $1/neural_compressor #> /lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then From 39d864baf40f05fd26d3fccfcc6858cb4e8d872b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:29:22 +0800 Subject: [PATCH 205/296] test --- .azure-pipelines/code-scan.yml | 78 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index c84f277c848..a0dcbaad1d8 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -21,47 +21,47 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - # - stage: BanditCodeScan - # displayName: Bandit Code Scan - # dependsOn: [] - # jobs: - # - job: Bandit - # displayName: Bandit - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" - # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" + docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # artifact: $(BANDIT_CONTAINER_NAME) - # publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + artifact: $(BANDIT_CONTAINER_NAME) + publishLocation: "pipeline" # - stage: PylintCodeScan # displayName: Pylint Code Scan From 25d90a6065efd3b028fc2f764cc7c46ebfd05e31 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:41:35 +0800 Subject: [PATCH 206/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 78f9ec1ce01..12d5614ff68 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -4,10 +4,11 @@ echo "--------------------" $1 echo "--------------------" -python -m bandit -r -lll -iii $1/neural_compressor #> /lpot-bandit.log +python -m bandit -r -lll -iii $1/neural_compressor > /lpot-bandit.log + exit_code=$? -if [ ${exit_code} -ne 0 ] ; then +if [ ${exit_code} -e 0 ] ; then echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 \ No newline at end of file From a4702924fbe327bc4f258f3221628a46b7542f05 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 17:44:32 +0800 Subject: [PATCH 207/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 12d5614ff68..b67fa590565 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -6,9 +6,9 @@ echo "--------------------" python -m bandit -r -lll -iii $1/neural_compressor > /lpot-bandit.log - +exit 1 exit_code=$? -if [ ${exit_code} -e 0 ] ; then +if [ ${exit_code} -ne 0 ] ; then echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 \ No newline at end of file From 7043b982b12af04f2ff8b0c106319d996f136f5c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 18:51:14 +0800 Subject: [PATCH 208/296] test --- .azure-pipelines/code-scan.yml | 10 +++++++--- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 4 ---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index a0dcbaad1d8..27d8c363ff1 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -63,6 +63,7 @@ stages: artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" + # - stage: PylintCodeScan # displayName: Pylint Code Scan # dependsOn: [] @@ -180,9 +181,12 @@ stages: inputs: targetType: "inline" script: | - echo "$(TARGET_BRANCH)" - echo "$(System.PullRequest.TargetBranch)" - echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" + echo ${BUILD_SOURCESDIRECTORY} + echo $(Build.SourcesDirectory) + echo $(CURRENT_PATH) + echo $(TARGET_BRANCH) + echo $(System.PullRequest.TargetBranch) + echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index b67fa590565..a3788248cf2 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,12 +1,8 @@ pip install bandit -echo "--------------------" -$1 -echo "--------------------" python -m bandit -r -lll -iii $1/neural_compressor > /lpot-bandit.log -exit 1 exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Bandit exited with non-zero exit code."; exit 1 From e886f5a3c2d18e38aeb0804c2e2741e318998b68 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 20:34:26 +0800 Subject: [PATCH 209/296] test --- .azure-pipelines/code-scan.yml | 101 ++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 27d8c363ff1..8c11ca6fa7d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -21,49 +21,49 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - - stage: BanditCodeScan - displayName: Bandit Code Scan - dependsOn: [] - jobs: - - job: Bandit - displayName: Bandit - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" - docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # - stage: BanditCodeScan + # displayName: Bandit Code Scan + # dependsOn: [] + # jobs: + # - job: Bandit + # displayName: Bandit + # steps: + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - artifact: $(BANDIT_CONTAINER_NAME) - publishLocation: "pipeline" + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" + # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # artifact: $(BANDIT_CONTAINER_NAME) + # publishLocation: "pipeline" + # - stage: PylintCodeScan # displayName: Pylint Code Scan # dependsOn: [] @@ -156,6 +156,9 @@ stages: displayName: CopyRight steps: - script: | + echo "--------Clean workspace-----------" + git show-ref -s remotes/origin/master + echo "-------------------------" echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true echo y | docker system prune @@ -163,6 +166,10 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | + echo "--------After Checkout-----------" + git show-ref -s remotes/origin/master + echo "-------------------------" + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi @@ -173,6 +180,10 @@ stages: fi displayName: "Build Devel Images" - script: | + echo "--------After Build-----------" + git show-ref -s remotes/origin/master + echo "-------------------------" + docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" @@ -181,12 +192,10 @@ stages: inputs: targetType: "inline" script: | - echo ${BUILD_SOURCESDIRECTORY} - echo $(Build.SourcesDirectory) - echo $(CURRENT_PATH) - echo $(TARGET_BRANCH) - echo $(System.PullRequest.TargetBranch) - echo ${SYSTEM_PULLREQUEST_TARGETBRANCH} + echo "$(TARGET_BRANCH)" + echo "$(System.PullRequest.TargetBranch)" + echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From d880175e71378d324f1d039251497fedb49739b2 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:07:18 +0800 Subject: [PATCH 210/296] test --- .azure-pipelines/code-scan.yml | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 8c11ca6fa7d..b4ff462eba6 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -156,9 +156,7 @@ stages: displayName: CopyRight steps: - script: | - echo "--------Clean workspace-----------" - git show-ref -s remotes/origin/master - echo "-------------------------" + echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true echo y | docker system prune @@ -166,10 +164,6 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - echo "--------After Checkout-----------" - git show-ref -s remotes/origin/master - echo "-------------------------" - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi @@ -180,10 +174,6 @@ stages: fi displayName: "Build Devel Images" - script: | - echo "--------After Build-----------" - git show-ref -s remotes/origin/master - echo "-------------------------" - docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" @@ -192,13 +182,17 @@ stages: inputs: targetType: "inline" script: | - echo "$(TARGET_BRANCH)" - echo "$(System.PullRequest.TargetBranch)" - echo "${SYSTEM_PULLREQUEST_TARGETBRANCH}" + echo "--------BEFORE docker Run-----------" + git show-ref -s remotes/origin/master + echo "-------------------------" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + echo "--------after docker Run-----------" + git show-ref -s remotes/origin/master + echo "-------------------------" + - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From a8a1e6b283db6eec96882da3135ca7866a647a03 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:09:01 +0800 Subject: [PATCH 211/296] test --- .azure-pipelines/code-scan.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index b4ff462eba6..f2d6e80d43a 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -186,6 +186,9 @@ stages: git show-ref -s remotes/origin/master echo "-------------------------" + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + echo "-------------------------" + docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From 7e68f214d19bc795b3503e6c5ab7ed040a23cab3 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:26:46 +0800 Subject: [PATCH 212/296] test --- .azure-pipelines/code-scan.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f2d6e80d43a..050a4d84e8f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -185,8 +185,13 @@ stages: echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master echo "-------------------------" + + pwd git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + + echo "----------------------------" + $(System.PullRequest.TargetBranch) echo "-------------------------" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " From 634aaa8ee8556f06e4328b335fb6b099fcf82be9 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:30:02 +0800 Subject: [PATCH 213/296] test --- .azure-pipelines/code-scan.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 050a4d84e8f..f218b53430b 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -184,15 +184,13 @@ stages: script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - echo "-------------------------" - - pwd + echo "-----------11--------------" git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor - echo "----------------------------" + echo "-----------1-----------------" $(System.PullRequest.TargetBranch) - echo "-------------------------" + echo "-----------1--------------" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From 048bd7f9ad6c1c712934e6ecb82bba0e1aee7c44 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:34:08 +0800 Subject: [PATCH 214/296] test --- .azure-pipelines/code-scan.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f218b53430b..49f4dd84ac7 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -156,7 +156,9 @@ stages: displayName: CopyRight steps: - script: | - + echo "--------Clean workspace-----------" + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + echo "-----------1-----------------" echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true echo y | docker system prune @@ -164,6 +166,9 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | + echo "--------Build Devel Images-----------" + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + echo "-----------1-----------------" if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi @@ -174,6 +179,9 @@ stages: fi displayName: "Build Devel Images" - script: | + echo "--------Clean Docker-----------" + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + echo "-----------1-----------------" docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" @@ -183,11 +191,7 @@ stages: targetType: "inline" script: | echo "--------BEFORE docker Run-----------" - git show-ref -s remotes/origin/master - echo "-----------11--------------" - - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor - + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor echo "-----------1-----------------" $(System.PullRequest.TargetBranch) echo "-----------1--------------" From b79757834bfb161f661ca4ce628d50525f4ffec7 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:41:34 +0800 Subject: [PATCH 215/296] test --- .azure-pipelines/code-scan.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 49f4dd84ac7..f908f7845f1 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -155,8 +155,9 @@ stages: - job: CopyRight displayName: CopyRight steps: - - script: | + - script: | echo "--------Clean workspace-----------" + git show-ref -s remotes/origin/master git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor echo "-----------1-----------------" echo ${BUILD_SOURCESDIRECTORY} From 16077c335e5ec2de320274738a7a884fb6115468 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:44:26 +0800 Subject: [PATCH 216/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f908f7845f1..6cd1e7b34bf 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -158,7 +158,7 @@ stages: - script: | echo "--------Clean workspace-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) ./neural_compressor echo "-----------1-----------------" echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true From 7013b49b61e1430221758cee4a3c6759d60e6bbb Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 21:55:59 +0800 Subject: [PATCH 217/296] test --- .azure-pipelines/code-scan.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 6cd1e7b34bf..1a5ab1fb661 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -156,10 +156,6 @@ stages: displayName: CopyRight steps: - script: | - echo "--------Clean workspace-----------" - git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) ./neural_compressor - echo "-----------1-----------------" echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true echo y | docker system prune @@ -167,9 +163,6 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - echo "--------Build Devel Images-----------" - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor - echo "-----------1-----------------" if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi @@ -180,9 +173,6 @@ stages: fi displayName: "Build Devel Images" - script: | - echo "--------Clean Docker-----------" - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor - echo "-----------1-----------------" docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" @@ -192,18 +182,14 @@ stages: targetType: "inline" script: | echo "--------BEFORE docker Run-----------" + git show-ref -s remotes/origin/master git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor - echo "-----------1-----------------" + echo "-----------pr-----------------" $(System.PullRequest.TargetBranch) - echo "-----------1--------------" + echo "-----------pr--------------" docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - - echo "--------after docker Run-----------" - git show-ref -s remotes/origin/master - echo "-------------------------" - - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log From a00566f059fa1e948ffa4f27f026b9f48f0a39b0 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 22:00:39 +0800 Subject: [PATCH 218/296] test --- .azure-pipelines/code-scan.yml | 2 +- .../scripts/codeScan/copyright/copy_right.sh | 52 +++++++++---------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 1a5ab1fb661..f0396af360f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -183,7 +183,7 @@ stages: script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) ./neural_compressor echo "-----------pr-----------------" $(System.PullRequest.TargetBranch) echo "-----------pr--------------" diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh index bbfbec73987..6187b3ebeb0 100644 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh @@ -1,35 +1,31 @@ - # $1: $(target_path) - # $2: $(TARGET_BRANCH) - $(System.PullRequest.TargetBranch) - # $3: $(VAL_PATH) - supported_extensions=(py, sh, yaml) - - set -xe - git config --global --add safe.directory /neural_compressor - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log - # git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log +supported_extensions=(py, sh, yaml) + +set -xe +git config --global --add safe.directory /neural_compressor +git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log +# git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log - files=$(cat /diff.log | awk '!a[$0]++') - # files=$(cat $3/copyright/diff.log | awk '!a[$0]++') +files=$(cat /diff.log | awk '!a[$0]++') - for file in ${files} +for file in ${files} - do - if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - echo "Checking license in 1 ${file}" - if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - echo ${file} >> /copyright_issue_summary.log - cat /copyright_issue_summary.log - fi - else - echo "Skipping ${file}" +do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + echo "Checking license in 1 ${file}" + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> /copyright_issue_summary.log + cat /copyright_issue_summary.log fi - done + else + echo "Skipping ${file}" + fi +done - ls /copyright_issue_summary.log - exit_code=$? - if [ ${exit_code} -e 0 ] ; then - echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 - fi - exit 0 \ No newline at end of file +ls /copyright_issue_summary.log +exit_code=$? +if [ ${exit_code} -e 0 ] ; then + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 +fi +exit 0 \ No newline at end of file From e3fbdcd7205f29d5fc9f687a44227b268e770964 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 22:03:22 +0800 Subject: [PATCH 219/296] test --- .azure-pipelines/code-scan.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f0396af360f..7f0b2065278 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -183,7 +183,8 @@ stages: script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) ./neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /$(CURRENT_PATH)/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /$(CURRENT_PATH)/neural_compressor/neural_compressor echo "-----------pr-----------------" $(System.PullRequest.TargetBranch) echo "-----------pr--------------" From cca691edf5558e2f9293c16a8d33e13f4b1efc73 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 22:05:11 +0800 Subject: [PATCH 220/296] test --- .azure-pipelines/code-scan.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 7f0b2065278..73c3bc550cf 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -183,8 +183,11 @@ stages: script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /$(CURRENT_PATH)/neural_compressor - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /$(CURRENT_PATH)/neural_compressor/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor/neural_compressor + echo "--------neural_compressor Path-----------" + cd $(CURRENT_PATH)/neural_compressor + ls echo "-----------pr-----------------" $(System.PullRequest.TargetBranch) echo "-----------pr--------------" From 1449e9bb8828dd6920ae0e044b668095e421f9dd Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 22:08:34 +0800 Subject: [PATCH 221/296] test --- .azure-pipelines/code-scan.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 73c3bc550cf..d73d1bdb86c 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -186,6 +186,7 @@ stages: git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor/neural_compressor echo "--------neural_compressor Path-----------" + echo $(CURRENT_PATH) cd $(CURRENT_PATH)/neural_compressor ls echo "-----------pr-----------------" From 0f84a6677c534d24865599190c1d3fc04123b9a7 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Thu, 11 Aug 2022 22:14:44 +0800 Subject: [PATCH 222/296] test --- .azure-pipelines/code-scan.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d73d1bdb86c..10fe47d4639 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -184,7 +184,6 @@ stages: echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor/neural_compressor echo "--------neural_compressor Path-----------" echo $(CURRENT_PATH) cd $(CURRENT_PATH)/neural_compressor From 25dfade2f602fc2734fb9bc94c205e521789d065 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 11 Aug 2022 22:44:29 +0800 Subject: [PATCH 223/296] for test --- .azure-pipelines/code-scan.yml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 10fe47d4639..33e3d4212c4 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,5 +1,4 @@ -trigger: - - code-scan +trigger: none pr: - master pool: suyue-test @@ -155,35 +154,14 @@ stages: - job: CopyRight displayName: CopyRight steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - + - - task: Bash@3 inputs: targetType: "inline" script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $CURRENT_PATH/neural_compressor echo "--------neural_compressor Path-----------" echo $(CURRENT_PATH) cd $(CURRENT_PATH)/neural_compressor From 6cef4bca0897f17d63b3a9783eb008169dade723 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 11 Aug 2022 22:48:01 +0800 Subject: [PATCH 224/296] fix --- .azure-pipelines/code-scan.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 33e3d4212c4..de29eb91783 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -154,7 +154,6 @@ stages: - job: CopyRight displayName: CopyRight steps: - - - task: Bash@3 inputs: targetType: "inline" From ecc62a5dee5c1f1af60ce92f7abcff1b38d19916 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 11 Aug 2022 22:52:13 +0800 Subject: [PATCH 225/296] debug --- .azure-pipelines/code-scan.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index de29eb91783..1035d91a942 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -154,13 +154,19 @@ stages: - job: CopyRight displayName: CopyRight steps: + - script: | + echo "--------run with script-----------" + git show-ref -s remotes/origin/master + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor + + - task: Bash@3 inputs: targetType: "inline" script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $CURRENT_PATH/neural_compressor + git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor echo "--------neural_compressor Path-----------" echo $(CURRENT_PATH) cd $(CURRENT_PATH)/neural_compressor From 9bd14f0a342c2205a1461215e2453027fe4cdb5d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 11 Aug 2022 23:16:30 +0800 Subject: [PATCH 226/296] test --- .azure-pipelines/code-scan.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 1035d91a942..d840dfd0086 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -154,23 +154,17 @@ stages: - job: CopyRight displayName: CopyRight steps: - - script: | - echo "--------run with script-----------" - git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor - - - task: Bash@3 inputs: targetType: "inline" script: | echo "--------BEFORE docker Run-----------" git show-ref -s remotes/origin/master - git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor + git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor echo "--------neural_compressor Path-----------" echo $(CURRENT_PATH) - cd $(CURRENT_PATH)/neural_compressor - ls + cd $(CURRENT_PATH) + ll -a echo "-----------pr-----------------" $(System.PullRequest.TargetBranch) echo "-----------pr--------------" From 242a54e23f6e0608450d3b9972b90398ae7f1aee Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 09:40:25 +0800 Subject: [PATCH 227/296] test --- .azure-pipelines/code-scan.yml | 48 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d840dfd0086..7a720da05c6 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -7,16 +7,15 @@ variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" - TARGET_PATH: "/neural_compressor" - CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" - CODE_SCAN_LOG_PATH: "scanLog" + # TARGET_PATH: "/neural_compressor" + CODE_SCAN_PATH: "/neural_compressor/.azure-pipelines/scripts/codeScan" + CODE_SCAN_LOG_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" BANDIT_CONTAINER_NAME: "bandit" PYLINT_CONTAINER_NAME: "pylint" PYSPELLING_CONTAINER_NAME: "pyspelling" COPYRIGHT_CONTAINER_NAME: "copyright" - TARGET_BRANCH: $(System.PullRequest.TargetBranch) CURRENT_PATH: $(Build.SourcesDirectory) stages: @@ -158,21 +157,32 @@ stages: inputs: targetType: "inline" script: | - echo "--------BEFORE docker Run-----------" - git show-ref -s remotes/origin/master - git --no-pager diff --name-only $(git show-ref -s remotes/origin/master) $(CURRENT_PATH)/neural_compressor - echo "--------neural_compressor Path-----------" - echo $(CURRENT_PATH) - cd $(CURRENT_PATH) - ll -a - echo "-----------pr-----------------" - $(System.PullRequest.TargetBranch) - echo "-----------pr--------------" - - docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(COPYRIGHT_CONTAINER_NAME)/copy_right.sh " - docker cp $(COPYRIGHT_CONTAINER_NAME):/copyright_issue_summary.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log + files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + + for file in ${files} + + do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + echo "Checking license in 1 ${file}" + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + fi + else + echo "Skipping ${file}" + fi + done + + + ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + exit_code=$? + if [ ${exit_code} -e 0 ] ; then + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + fi + exit 0 + - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - artifact: $(PYSPELLING_CONTAINER_NAME) + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + artifact: $(COPYRIGHT_CONTAINER_NAME) publishLocation: "pipeline" \ No newline at end of file From e304eda0e97f8be8f6067dbe3c27ed239e8daefc Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Fri, 12 Aug 2022 09:43:43 +0800 Subject: [PATCH 228/296] copyright test --- neural_compressor/objective.py | 1 - 1 file changed, 1 deletion(-) diff --git a/neural_compressor/objective.py b/neural_compressor/objective.py index b6e656ff0a9..a7561cbd02b 100644 --- a/neural_compressor/objective.py +++ b/neural_compressor/objective.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (c) 2021 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From a4efa3ece8bdbe02f1ac0955dedbd5840c835fd6 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 09:47:05 +0800 Subject: [PATCH 229/296] test --- .azure-pipelines/code-scan.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 7a720da05c6..91012c5ad6f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -157,6 +157,13 @@ stages: inputs: targetType: "inline" script: | + set -ex + echo "---------------------------" + ehco $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + ls + echo "---------------------------" + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') From 1c46a43fd69afcd5049a13cdeb8e69cbb4185579 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 09:48:21 +0800 Subject: [PATCH 230/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 91012c5ad6f..13358f0e9c2 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -159,7 +159,7 @@ stages: script: | set -ex echo "---------------------------" - ehco $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + echo $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) ls echo "---------------------------" From fc7b05e525d02e3e136e90bd5d746e803d3470bd Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 09:49:25 +0800 Subject: [PATCH 231/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 13358f0e9c2..685ac06cdd6 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -8,8 +8,8 @@ variables: IMAGE_TAG: "1.0" # TARGET_PATH: "/neural_compressor" - CODE_SCAN_PATH: "/neural_compressor/.azure-pipelines/scripts/codeScan" - CODE_SCAN_LOG_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" + CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" + CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" BANDIT_CONTAINER_NAME: "bandit" PYLINT_CONTAINER_NAME: "pylint" From 4b942552ab6a6d885d9111bfea1f547615513d27 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 09:51:54 +0800 Subject: [PATCH 232/296] test --- .azure-pipelines/code-scan.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 685ac06cdd6..480a0f88b45 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -164,6 +164,8 @@ stages: ls echo "---------------------------" + supported_extensions=(py, sh, yaml) + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') From 3a36f278d620dfcea915cf40fbf8fa60e4d403bb Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Fri, 12 Aug 2022 09:53:23 +0800 Subject: [PATCH 233/296] Update code-scan.yml for Azure Pipelines --- .azure-pipelines/code-scan.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 480a0f88b45..97553b57958 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -158,19 +158,12 @@ stages: targetType: "inline" script: | set -ex - echo "---------------------------" - echo $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) - cd $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) - ls - echo "---------------------------" - supported_extensions=(py, sh, yaml) git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') for file in ${files} - do if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then echo "Checking license in 1 ${file}" @@ -182,7 +175,6 @@ stages: fi done - ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log exit_code=$? if [ ${exit_code} -e 0 ] ; then From f51756f686e1694ac0dc0383a64e9dde45776ce8 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 09:58:29 +0800 Subject: [PATCH 234/296] test --- .azure-pipelines/code-scan.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 97553b57958..ce9fffebe16 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -176,8 +176,7 @@ stages: done ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - exit_code=$? - if [ ${exit_code} -e 0 ] ; then + if [ $? -eq 0 ] ; then echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 fi exit 0 From b8a68b76de272d0e5f17e2900b4598c3875f06fa Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:00:51 +0800 Subject: [PATCH 235/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index ce9fffebe16..cfa56e11718 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,4 +1,4 @@ -trigger: none +trigger: code-scan pr: - master pool: suyue-test From 864dcca7a1dfaec1634cfac16ccbf6affe71cc50 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:01:28 +0800 Subject: [PATCH 236/296] test --- .azure-pipelines/code-scan.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index cfa56e11718..15cb9d0406f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,4 +1,5 @@ -trigger: code-scan +trigger: + - code-scan pr: - master pool: suyue-test From dccc7adcffbd3000129935d6421762710abb38d6 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:03:33 +0800 Subject: [PATCH 237/296] TEST --- .azure-pipelines/code-scan.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 15cb9d0406f..d8e7f43b742 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -176,14 +176,18 @@ stages: fi done - ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - if [ $? -eq 0 ] ; then - echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 - fi - exit 0 - - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log artifact: $(COPYRIGHT_CONTAINER_NAME) - publishLocation: "pipeline" \ No newline at end of file + publishLocation: "pipeline" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + if [ $? -eq 0 ] ; then + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + fi + exit 0 \ No newline at end of file From 7bb6ee76ce58e36092808c633feea412ae42c058 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:36:07 +0800 Subject: [PATCH 238/296] test --- .azure-pipelines/code-scan.yml | 89 ++++++++++--------- .../scripts/codeScan/bandit/bandit.sh | 9 +- 2 files changed, 54 insertions(+), 44 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d8e7f43b742..899a403cb79 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,5 +1,4 @@ -trigger: - - code-scan +trigger: none pr: - master pool: suyue-test @@ -11,6 +10,8 @@ variables: # TARGET_PATH: "/neural_compressor" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" + DOCKER_CODE_SCAN_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan" + BANDIT_CONTAINER_NAME: "bandit" PYLINT_CONTAINER_NAME: "pylint" @@ -20,47 +21,51 @@ variables: CURRENT_PATH: $(Build.SourcesDirectory) stages: - # - stage: BanditCodeScan - # displayName: Bandit Code Scan - # dependsOn: [] - # jobs: - # - job: Bandit - # displayName: Bandit - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" - - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh $(TARGET_PATH)" - # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # artifact: $(BANDIT_CONTAINER_NAME) - # publishLocation: "pipeline" + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash $(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh + # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + echo "---------$bandit_log_dir-------------" + echo $bandit_log_dir + echo "---------$(bandit_log_dir)-------------" + echo $(bandit_log_dir) + echo "----------------------" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$bandit_log_dir/lpot-bandit.log + artifact: $(BANDIT_CONTAINER_NAME) + publishLocation: "pipeline" # - stage: PylintCodeScan diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index a3788248cf2..b52ded2b6a8 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,10 +1,15 @@ +set -ex +bandit_log_dir = "/neural-compressor/neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" + pip install bandit -python -m bandit -r -lll -iii $1/neural_compressor > /lpot-bandit.log +python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Bandit exited with non-zero exit code."; exit 1 fi -exit 0 \ No newline at end of file +exit 0 + +export bandit_log_dir \ No newline at end of file From fbb3faf5d29aa26b0cb03a9921d7094544abb6db Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:39:06 +0800 Subject: [PATCH 239/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 899a403cb79..f2ed986be47 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -54,7 +54,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash $(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh + docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log echo "---------$bandit_log_dir-------------" echo $bandit_log_dir @@ -63,7 +63,7 @@ stages: echo "----------------------" - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH)/$bandit_log_dir/lpot-bandit.log + targetPath: $(CURRENT_PATH)/$(DOCKER_CODE_SCAN_PATH)/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" From 08f685305d818bddcfabebb071dd7f9f829f4d3b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:39:34 +0800 Subject: [PATCH 240/296] test --- .azure-pipelines/code-scan.yml | 84 +++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f2ed986be47..07b0bf896c9 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -152,47 +152,47 @@ stages: # artifact: $(PYSPELLING_CONTAINER_NAME) # publishLocation: "pipeline" - - stage: CopyRight - displayName: CopyRight Code Scan - dependsOn: [] - jobs: - - job: CopyRight - displayName: CopyRight - steps: - - task: Bash@3 - inputs: - targetType: "inline" - script: | - set -ex - supported_extensions=(py, sh, yaml) - - git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log - files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') - - for file in ${files} - do - if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - echo "Checking license in 1 ${file}" - if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - fi - else - echo "Skipping ${file}" - fi - done + # - stage: CopyRight + # displayName: CopyRight Code Scan + # dependsOn: [] + # jobs: + # - job: CopyRight + # displayName: CopyRight + # steps: + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # set -ex + # supported_extensions=(py, sh, yaml) + + # git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log + # files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + + # for file in ${files} + # do + # if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + # echo "Checking license in 1 ${file}" + # if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + # echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # fi + # else + # echo "Skipping ${file}" + # fi + # done - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - artifact: $(COPYRIGHT_CONTAINER_NAME) - publishLocation: "pipeline" + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # artifact: $(COPYRIGHT_CONTAINER_NAME) + # publishLocation: "pipeline" - - task: Bash@3 - inputs: - targetType: "inline" - script: | - ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - if [ $? -eq 0 ] ; then - echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 - fi - exit 0 \ No newline at end of file + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # if [ $? -eq 0 ] ; then + # echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + # fi + # exit 0 \ No newline at end of file From fa4e279fd770a463bba1f6129eebb988fd0f325d Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:41:59 +0800 Subject: [PATCH 241/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 07b0bf896c9..9c0db2dc171 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -10,7 +10,7 @@ variables: # TARGET_PATH: "/neural_compressor" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" - DOCKER_CODE_SCAN_PATH: "neural_compressor/.azure-pipelines/scripts/codeScan" + DOCKER_CODE_SCAN_PATH: "neural-compressor/.azure-pipelines/scripts/codeScan" BANDIT_CONTAINER_NAME: "bandit" From 88d746bd409ddd45b43e64215962a10e53d5a45f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:45:22 +0800 Subject: [PATCH 242/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index b52ded2b6a8..9f11587a694 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,5 +1,5 @@ set -ex -bandit_log_dir = "/neural-compressor/neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" +bandit_log_dir="/neural-compressor/neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install bandit @@ -12,4 +12,4 @@ if [ ${exit_code} -ne 0 ] ; then fi exit 0 -export bandit_log_dir \ No newline at end of file + \ No newline at end of file From 389368413b011557dad21c8a3d1d16c01eef8a6a Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:46:56 +0800 Subject: [PATCH 243/296] test --- .azure-pipelines/code-scan.yml | 5 ----- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9c0db2dc171..d5e385d0c96 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -56,11 +56,6 @@ stages: script: | docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - echo "---------$bandit_log_dir-------------" - echo $bandit_log_dir - echo "---------$(bandit_log_dir)-------------" - echo $(bandit_log_dir) - echo "----------------------" - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(DOCKER_CODE_SCAN_PATH)/lpot-bandit.log diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 9f11587a694..544040617a9 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,5 +1,5 @@ set -ex -bandit_log_dir="/neural-compressor/neural_compressor/.azure-pipelines/scripts/codeScan/scanLog" +bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install bandit @@ -12,4 +12,3 @@ if [ ${exit_code} -ne 0 ] ; then fi exit 0 - \ No newline at end of file From fb3583ad387814022f720533d1131cd039c2f369 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:50:21 +0800 Subject: [PATCH 244/296] test --- .azure-pipelines/code-scan.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d5e385d0c96..a26b0804dd6 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -55,6 +55,15 @@ stages: targetType: "inline" script: | docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh + echo "-----------$(CURRENT_PATH)/neural-compressor------------------" + cd $(CURRENT_PATH)/neural-compressor + ls + echo "-----------------------------" + + echo "-----------$(CURRENT_PATH)------------------" + cd $(CURRENT_PATH) + ls + echo "-----------------------------" # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: From 1aec85cc43a345fea571eee95cb44eb172609902 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 10:56:43 +0800 Subject: [PATCH 245/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index a26b0804dd6..38cee731c4a 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -56,7 +56,7 @@ stages: script: | docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh echo "-----------$(CURRENT_PATH)/neural-compressor------------------" - cd $(CURRENT_PATH)/neural-compressor + cd $(CURRENT_PATH)/docker ls echo "-----------------------------" From 1700de4aa82fa816090fdcb1324eb699613b5808 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:00:33 +0800 Subject: [PATCH 246/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 38cee731c4a..95f42dc761b 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -56,7 +56,7 @@ stages: script: | docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh echo "-----------$(CURRENT_PATH)/neural-compressor------------------" - cd $(CURRENT_PATH)/docker + cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan/scanLog ls echo "-----------------------------" From 37827dbaaa6a8426699be152a245abfa17600748 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:02:55 +0800 Subject: [PATCH 247/296] test --- .azure-pipelines/code-scan.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 95f42dc761b..98b286aafc3 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -55,19 +55,10 @@ stages: targetType: "inline" script: | docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh - echo "-----------$(CURRENT_PATH)/neural-compressor------------------" - cd $(CURRENT_PATH)/.azure-pipelines/scripts/codeScan/scanLog - ls - echo "-----------------------------" - - echo "-----------$(CURRENT_PATH)------------------" - cd $(CURRENT_PATH) - ls - echo "-----------------------------" # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH)/$(DOCKER_CODE_SCAN_PATH)/lpot-bandit.log + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log artifact: $(BANDIT_CONTAINER_NAME) publishLocation: "pipeline" From 34e50d8e82976dedeb9fc0acb0892ae42c9c2edb Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:14:31 +0800 Subject: [PATCH 248/296] test --- .azure-pipelines/code-scan.yml | 108 ++++++++---------- .../scripts/codeScan/pylint/pylint.sh | 6 +- 2 files changed, 53 insertions(+), 61 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 98b286aafc3..62ba2a7c1c4 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -10,65 +10,16 @@ variables: # TARGET_PATH: "/neural_compressor" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" - DOCKER_CODE_SCAN_PATH: "neural-compressor/.azure-pipelines/scripts/codeScan" - - - BANDIT_CONTAINER_NAME: "bandit" - PYLINT_CONTAINER_NAME: "pylint" - PYSPELLING_CONTAINER_NAME: "pyspelling" - COPYRIGHT_CONTAINER_NAME: "copyright" CURRENT_PATH: $(Build.SourcesDirectory) stages: - - stage: BanditCodeScan - displayName: Bandit Code Scan - dependsOn: [] - jobs: - - job: Bandit - displayName: Bandit - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name=$(BANDIT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /$(DOCKER_CODE_SCAN_PATH)/$(BANDIT_CONTAINER_NAME)/bandit.sh - # docker cp $(BANDIT_CONTAINER_NAME):/lpot-bandit.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - artifact: $(BANDIT_CONTAINER_NAME) - publishLocation: "pipeline" - - - # - stage: PylintCodeScan - # displayName: Pylint Code Scan + # - stage: BanditCodeScan + # displayName: Bandit Code Scan # dependsOn: [] # jobs: - # - job: Pylint - # displayName: Pylint + # - job: Bandit + # displayName: Bandit # steps: # - script: | # echo ${BUILD_SOURCESDIRECTORY} @@ -96,15 +47,56 @@ stages: # inputs: # targetType: "inline" # script: | - # docker run --disable-content-trust --privileged --name=$(PYLINT_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYLINT_CONTAINER_NAME)/pylint.sh $(TARGET_PATH)" - # docker cp $(PYLINT_CONTAINER_NAME):/lpot-pylint.json $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - + # docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh + # - task: PublishPipelineArtifact@1 # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - # artifact: $(PYLINT_CONTAINER_NAME) + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + # artifact: bandit # publishLocation: "pipeline" + + - stage: PylintCodeScan + displayName: Pylint Code Scan + dependsOn: [] + jobs: + - job: Pylint + displayName: Pylint + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.log + artifact: pylint + publishLocation: "pipeline" + # - stage: PyspellingCodeScan # displayName: Pyspelling Code Scan # dependsOn: [] diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index a6ff9db0999..7026071c2d1 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -1,5 +1,5 @@ -# $1: $(VAL_PATH) -# $2: $(target_path) +set -ex +pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install pylint==2.12.1 pip install flask==2.1.3 @@ -16,7 +16,7 @@ pip install onnxruntime_extensions -python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime $1/$1 > /lpot-pylint.json +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json exit_code=$? if [ ${exit_code} -ne 0 ] ; then From bc4a783569c3bc3a0e4b9d9b4f4780af069eb83f Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:18:00 +0800 Subject: [PATCH 249/296] test --- .azure-pipelines/scripts/codeScan/pylint/pylint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 7026071c2d1..cef8620dae5 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -11,7 +11,7 @@ pip install autograd pip install pymoo pip install tf_slim pip install transformers -pip install -r requirements.txt +pip install -r /neural-compressor/requirements.txt pip install onnxruntime_extensions From eaec1bde90be83a6c9c80368c919fa59bc83aced Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:23:05 +0800 Subject: [PATCH 250/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 62ba2a7c1c4..baaf867a2ff 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -93,7 +93,7 @@ stages: - task: PublishPipelineArtifact@1 inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.log + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json artifact: pylint publishLocation: "pipeline" From 7fa9242e9909c26f74f330ea964e58fc15de5c07 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:41:13 +0800 Subject: [PATCH 251/296] test --- .azure-pipelines/code-scan.yml | 101 +++++++++--------- .../scripts/codeScan/pyspelling/pyspelling.sh | 13 ++- 2 files changed, 58 insertions(+), 56 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index baaf867a2ff..4ba39ec9048 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -56,53 +56,12 @@ stages: # publishLocation: "pipeline" - - stage: PylintCodeScan - displayName: Pylint Code Scan - dependsOn: [] - jobs: - - job: Pylint - displayName: Pylint - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - artifact: pylint - publishLocation: "pipeline" - - # - stage: PyspellingCodeScan - # displayName: Pyspelling Code Scan + # - stage: PylintCodeScan + # displayName: Pylint Code Scan # dependsOn: [] # jobs: - # - job: Pyspelling - # displayName: Pyspelling + # - job: Pylint + # displayName: Pylint # steps: # - script: | # echo ${BUILD_SOURCESDIRECTORY} @@ -130,14 +89,54 @@ stages: # inputs: # targetType: "inline" # script: | - # docker run --disable-content-trust --privileged --name=$(PYSPELLING_CONTAINER_NAME) -v ${BUILD_SOURCESDIRECTORY}:$(TARGET_PATH) ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd $(TARGET_PATH) && bash $(TARGET_PATH)/$(CODE_SCAN_PATH)/$(PYSPELLING_CONTAINER_NAME)/pyspelling.sh $(TARGET_PATH)/$(CODE_SCAN_PATH)" - # docker cp $(PYSPELLING_CONTAINER_NAME):/lpot_pyspelling.log $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - + # docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh + # - task: PublishPipelineArtifact@1 # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - # artifact: $(PYSPELLING_CONTAINER_NAME) - # publishLocation: "pipeline" + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # artifact: pylint + # publishLocation: "pipeline" + + - stage: PyspellingCodeScan + displayName: Pyspelling Code Scan + dependsOn: [] + jobs: + - job: Pyspelling + displayName: Pyspelling + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + artifact: pyspelling + publishLocation: "pipeline" # - stage: CopyRight # displayName: CopyRight Code Scan diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index f03e726c73c..2d64786d74c 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -1,3 +1,7 @@ +set -ex +pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" +pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" + pip install pyspelling pip install -r requirements.txt @@ -6,11 +10,10 @@ apt-get install aspell-en -y -sed -i "s|\${VAL_REPO}|$1|g" /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml -echo "Modified config:" -cat /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml -pyspelling -c /neural_compressor/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling_conf.yaml > /lpot_pyspelling.log +sed -i "s|\${VAL_REPO}|$1|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|.|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml + +pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then echo "Pyspelling exited with non-zero exit code."; exit 1 From a4a952fc36b4d221268ed6a651650cc5e3c94cef Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:43:03 +0800 Subject: [PATCH 252/296] test --- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 2d64786d74c..845ccd20377 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -3,7 +3,7 @@ pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install pyspelling -pip install -r requirements.txt +pip install -r /neural-compressor/requirements.txt apt-get install aspell -y apt-get install aspell-en -y From cdca6deea0a63ebd95d70cc8d16c624e5c725676 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 11:44:44 +0800 Subject: [PATCH 253/296] test --- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 845ccd20377..1b2268384c0 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -10,7 +10,7 @@ apt-get install aspell-en -y -sed -i "s|\${VAL_REPO}|$1|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml +sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml sed -i "s|\${LPOT_REPO}|.|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log From 6cbb390102cf648bd7f16c6e3679abc041819bb1 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 12:28:18 +0800 Subject: [PATCH 254/296] test --- .azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 1b2268384c0..73b431ed8e4 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -11,7 +11,7 @@ apt-get install aspell-en -y sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml -sed -i "s|\${LPOT_REPO}|.|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml +sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log exit_code=$? From 5ae14a9489a19bc353f59cbd86fcf3fd232ba2a6 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 12:42:39 +0800 Subject: [PATCH 255/296] test --- .azure-pipelines/code-scan.yml | 234 ++++++++++++++++----------------- 1 file changed, 116 insertions(+), 118 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 4ba39ec9048..9aa7571492c 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -7,95 +7,93 @@ variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" - # TARGET_PATH: "/neural_compressor" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" - CURRENT_PATH: $(Build.SourcesDirectory) stages: - # - stage: BanditCodeScan - # displayName: Bandit Code Scan - # dependsOn: [] - # jobs: - # - job: Bandit - # displayName: Bandit - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" + - stage: BanditCodeScan + displayName: Bandit Code Scan + dependsOn: [] + jobs: + - job: Bandit + displayName: Bandit + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - # artifact: bandit - # publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log + artifact: bandit + publishLocation: "pipeline" - # - stage: PylintCodeScan - # displayName: Pylint Code Scan - # dependsOn: [] - # jobs: - # - job: Pylint - # displayName: Pylint - # steps: - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" + - stage: PylintCodeScan + displayName: Pylint Code Scan + dependsOn: [] + jobs: + - job: Pylint + displayName: Pylint + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - # artifact: pylint - # publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + artifact: pylint + publishLocation: "pipeline" - stage: PyspellingCodeScan displayName: Pyspelling Code Scan @@ -138,47 +136,47 @@ stages: artifact: pyspelling publishLocation: "pipeline" - # - stage: CopyRight - # displayName: CopyRight Code Scan - # dependsOn: [] - # jobs: - # - job: CopyRight - # displayName: CopyRight - # steps: - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # set -ex - # supported_extensions=(py, sh, yaml) + - stage: CopyRight + displayName: CopyRight Code Scan + dependsOn: [] + jobs: + - job: CopyRight + displayName: CopyRight + steps: + - task: Bash@3 + inputs: + targetType: "inline" + script: | + set -ex + supported_extensions=(py, sh, yaml) - # git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log - # files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log + files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') - # for file in ${files} - # do - # if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - # echo "Checking license in 1 ${file}" - # if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - # echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - # fi - # else - # echo "Skipping ${file}" - # fi - # done + for file in ${files} + do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + echo "Checking license in 1 ${file}" + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + fi + else + echo "Skipping ${file}" + fi + done - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - # artifact: $(COPYRIGHT_CONTAINER_NAME) - # publishLocation: "pipeline" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + artifact: $(COPYRIGHT_CONTAINER_NAME) + publishLocation: "pipeline" - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - # if [ $? -eq 0 ] ; then - # echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 - # fi - # exit 0 \ No newline at end of file + - task: Bash@3 + inputs: + targetType: "inline" + script: | + ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + if [ $? -eq 0 ] ; then + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + fi + exit 0 \ No newline at end of file From e739aef3ef5a96da837f6a95304b6fab7a56f368 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 12:45:32 +0800 Subject: [PATCH 256/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9aa7571492c..9eddea96795 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -168,7 +168,7 @@ stages: - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - artifact: $(COPYRIGHT_CONTAINER_NAME) + artifact: copyright publishLocation: "pipeline" - task: Bash@3 From 3da4e4fc089999a2e8455d30eaf2812d36468ed1 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 12:59:42 +0800 Subject: [PATCH 257/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 544040617a9..4436af6f342 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -7,7 +7,7 @@ pip install bandit python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log exit_code=$? -if [ ${exit_code} -ne 0 ] ; then +if [ ${exit_code} -eq 0 ] ; then echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 From 6de9c5179209b48e3b8f28e3a81f401546592996 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:24:27 +0800 Subject: [PATCH 258/296] test --- .azure-pipelines/code-scan.yml | 269 ++++++++++-------- .../scripts/codeScan/bandit/bandit.sh | 3 +- 2 files changed, 150 insertions(+), 122 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 9eddea96795..576783e464a 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -6,10 +6,10 @@ pool: suyue-test variables: IMAGE_NAME: "code-scan" IMAGE_TAG: "1.0" - CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) + CURRENT_STATUS: 0 stages: - stage: BanditCodeScan @@ -18,7 +18,13 @@ stages: jobs: - job: Bandit displayName: Bandit + continueOnError: true steps: + - script: | + if [ $(CURRENT_STATUS) -ne 0 ] ; then + exit 1 + fi + exit 0 - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true @@ -45,7 +51,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) - task: PublishPipelineArtifact@1 inputs: @@ -53,130 +59,153 @@ stages: artifact: bandit publishLocation: "pipeline" + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + + - - stage: PylintCodeScan - displayName: Pylint Code Scan - dependsOn: [] - jobs: - - job: Pylint - displayName: Pylint - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" + # - stage: PylintCodeScan + # displayName: Pylint Code Scan + # dependsOn: [] + # jobs: + # - job: Pylint + # displayName: Pylint + # steps: + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - artifact: pylint - publishLocation: "pipeline" + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + # artifact: pylint + # publishLocation: "pipeline" - - stage: PyspellingCodeScan - displayName: Pyspelling Code Scan - dependsOn: [] - jobs: - - job: Pyspelling - displayName: Pyspelling - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh + # - stage: PyspellingCodeScan + # displayName: Pyspelling Code Scan + # dependsOn: [] + # jobs: + # - job: Pyspelling + # displayName: Pyspelling + # steps: + # - script: | + # if [ $(CURRENT_STATUS) -ne 0 ] ; then + # exit 1 + # fi + # exit 0 + # - script: | + # echo ${BUILD_SOURCESDIRECTORY} + # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + # echo y | docker system prune + # displayName: "Clean workspace" + # - checkout: self + # displayName: "Checkout out Repo" + # - script: | + # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + # fi + # docker images | grep -i ${IMAGE_NAME} + # if [[ $? -ne 0 ]]; then + # echo "NO Such Image ${IMAGE_NAME}" + # exit 1 + # fi + # displayName: "Build Devel Images" + # - script: | + # docker stop $(docker ps -aq) + # docker rm -vf $(docker ps -aq) || true + # displayName: "Clean Docker" - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - artifact: pyspelling - publishLocation: "pipeline" + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh - - stage: CopyRight - displayName: CopyRight Code Scan - dependsOn: [] - jobs: - - job: CopyRight - displayName: CopyRight - steps: - - task: Bash@3 - inputs: - targetType: "inline" - script: | - set -ex - supported_extensions=(py, sh, yaml) - - git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log - files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') - - for file in ${files} - do - if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - echo "Checking license in 1 ${file}" - if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - fi - else - echo "Skipping ${file}" - fi - done + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + # artifact: pyspelling + # publishLocation: "pipeline" - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - artifact: copyright - publishLocation: "pipeline" + # - stage: CopyRight + # displayName: CopyRight Code Scan + # dependsOn: [] + # jobs: + # - job: CopyRight + # displayName: CopyRight + # steps: + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # set -ex + # supported_extensions=(py, sh, yaml) - - task: Bash@3 - inputs: - targetType: "inline" - script: | - ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - if [ $? -eq 0 ] ; then - echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 - fi - exit 0 \ No newline at end of file + # git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log + # files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + + # for file in ${files} + # do + # if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + # echo "Checking license in 1 ${file}" + # if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + # echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # fi + # else + # echo "Skipping ${file}" + # fi + # done + + # - task: PublishPipelineArtifact@1 + # inputs: + # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # artifact: copyright + # publishLocation: "pipeline" + + # - task: Bash@3 + # inputs: + # targetType: "inline" + # script: | + # ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + # if [ $? -eq 0 ] ; then + # echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + # fi + # exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 4436af6f342..27cffc3edb2 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,13 +1,12 @@ set -ex bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" - pip install bandit - python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log exit_code=$? if [ ${exit_code} -eq 0 ] ; then + $1 = 1; echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 From 9eb8b2f543073fd9487931e10b0768a0b0f297c9 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:27:26 +0800 Subject: [PATCH 259/296] test --- .azure-pipelines/code-scan.yml | 2 +- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 576783e464a..7b62b6fe89f 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -51,7 +51,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh - task: PublishPipelineArtifact@1 inputs: diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 27cffc3edb2..bccba11bd28 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -6,7 +6,6 @@ python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_lo exit_code=$? if [ ${exit_code} -eq 0 ] ; then - $1 = 1; echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 From c21efdff7e75bd474b4ca06cee9cbdb801697725 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:33:38 +0800 Subject: [PATCH 260/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 7b62b6fe89f..7c8b401fb1c 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -18,7 +18,7 @@ stages: jobs: - job: Bandit displayName: Bandit - continueOnError: true + condition: false steps: - script: | if [ $(CURRENT_STATUS) -ne 0 ] ; then From 87cf1046dd046d6e923d6c1f8735d36ae6e99e3b Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:36:05 +0800 Subject: [PATCH 261/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 7c8b401fb1c..2b274402dbd 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,7 +9,7 @@ variables: CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) - CURRENT_STATUS: 0 + CURRENT_STATUS: true stages: - stage: BanditCodeScan @@ -18,7 +18,7 @@ stages: jobs: - job: Bandit displayName: Bandit - condition: false + condition: $(CURRENT_STATUS) steps: - script: | if [ $(CURRENT_STATUS) -ne 0 ] ; then From 17f299edcef5e2efc68d35dddf3263f2f0847497 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:41:10 +0800 Subject: [PATCH 262/296] test --- .azure-pipelines/code-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 2b274402dbd..ef05e295b0d 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -18,7 +18,7 @@ stages: jobs: - job: Bandit displayName: Bandit - condition: $(CURRENT_STATUS) + condition: variables['CURRENT_STATUS'] steps: - script: | if [ $(CURRENT_STATUS) -ne 0 ] ; then From ed8700b49469fb9496135ab664a459e2a3f6dc6e Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:47:45 +0800 Subject: [PATCH 263/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index ef05e295b0d..4957fa129ea 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,7 +9,7 @@ variables: CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) - CURRENT_STATUS: true + CURRENT_STATUS: 'false' stages: - stage: BanditCodeScan @@ -18,7 +18,7 @@ stages: jobs: - job: Bandit displayName: Bandit - condition: variables['CURRENT_STATUS'] + condition: and(succeeded(), eq(variables.CURRENT_PATH, 'true')) steps: - script: | if [ $(CURRENT_STATUS) -ne 0 ] ; then From 37279fcd9c6fd4488f72c5565fcae5b039847e18 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:48:52 +0800 Subject: [PATCH 264/296] test --- .azure-pipelines/code-scan.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 4957fa129ea..cc304516f97 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -18,13 +18,8 @@ stages: jobs: - job: Bandit displayName: Bandit - condition: and(succeeded(), eq(variables.CURRENT_PATH, 'true')) + condition: eq(variables['CURRENT_STATUS'], 'true') steps: - - script: | - if [ $(CURRENT_STATUS) -ne 0 ] ; then - exit 1 - fi - exit 0 - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true From d56968d6febb205fd13b5f75cfa3f25e85244469 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 13:54:46 +0800 Subject: [PATCH 265/296] test --- .azure-pipelines/code-scan.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index cc304516f97..f581e1f3787 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,7 +9,7 @@ variables: CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) - CURRENT_STATUS: 'false' + CURRENT_STATUS: 'true' stages: - stage: BanditCodeScan @@ -46,7 +46,7 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) - task: PublishPipelineArtifact@1 inputs: From 5e747880de3e5bb4020a5b916221a49a7d4d2a0c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 14:06:40 +0800 Subject: [PATCH 266/296] test --- .azure-pipelines/code-scan.yml | 6 ++++-- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index f581e1f3787..91fae6f2749 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -9,7 +9,7 @@ variables: CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) - CURRENT_STATUS: 'true' + CURRENT_STATUS: true stages: - stage: BanditCodeScan @@ -47,7 +47,9 @@ stages: targetType: "inline" script: | docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) - + echo "------$(CURRENT_STATUS)-------" + echo $(CURRENT_STATUS) + echo "-------------" - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index bccba11bd28..58247134251 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,4 +1,4 @@ -set -ex +# set -ex bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install bandit @@ -6,6 +6,7 @@ python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_lo exit_code=$? if [ ${exit_code} -eq 0 ] ; then + sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 From 90ff87f3b6671804868c890973b795a337fc2ce1 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 14:09:14 +0800 Subject: [PATCH 267/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 58247134251..30bcbbcd1f1 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -6,7 +6,9 @@ python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_lo exit_code=$? if [ ${exit_code} -eq 0 ] ; then - sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml + sed -i "s|CURRENT_STATUS:.*|CURRENT_STATUS: false|g" /neural-compressor/azure-pipelines.yml + + # sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 From 3b81d2c408a2d4c6b6fe6e79c519f139e81190b9 Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 14:12:13 +0800 Subject: [PATCH 268/296] test --- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 30bcbbcd1f1..9a943a8ae00 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -6,7 +6,7 @@ python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_lo exit_code=$? if [ ${exit_code} -eq 0 ] ; then - sed -i "s|CURRENT_STATUS:.*|CURRENT_STATUS: false|g" /neural-compressor/azure-pipelines.yml + sed -i "s|CURRENT_STATUS\: true|CURRENT_STATUS\: false|g" /neural-compressor/azure-pipelines.yml # sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml echo "Bandit exited with non-zero exit code."; exit 1 From 86cd3fbcfcf487cbfb4784e0dc021dd5b067965c Mon Sep 17 00:00:00 2001 From: "Yue, Wenjiao" Date: Fri, 12 Aug 2022 14:15:34 +0800 Subject: [PATCH 269/296] test --- .azure-pipelines/code-scan.yml | 4 +++- .azure-pipelines/scripts/codeScan/bandit/bandit.sh | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 91fae6f2749..4cb89626c89 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -10,6 +10,7 @@ variables: CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) CURRENT_STATUS: true + ER: "1212" stages: - stage: BanditCodeScan @@ -49,7 +50,8 @@ stages: docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) echo "------$(CURRENT_STATUS)-------" echo $(CURRENT_STATUS) - echo "-------------" + + echo "---$(ER)----$(QR)------" - task: PublishPipelineArtifact@1 inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index 9a943a8ae00..e8f868a6f3f 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -7,6 +7,7 @@ python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_lo exit_code=$? if [ ${exit_code} -eq 0 ] ; then sed -i "s|CURRENT_STATUS\: true|CURRENT_STATUS\: false|g" /neural-compressor/azure-pipelines.yml + sed -i "s|ER|QR|g" /neural-compressor/azure-pipelines.yml # sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml echo "Bandit exited with non-zero exit code."; exit 1 From c1101be36164f0b8c5438949d9ad74d7f68c7769 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Tue, 16 Aug 2022 20:35:10 +0800 Subject: [PATCH 270/296] add code scan function (#21) --- .azure-pipelines/code-scan.yml | 302 +++++++++--------- .../scripts/codeScan/bandit/bandit.sh | 13 +- .../scripts/codeScan/copyright/copy_right.sh | 31 -- .../scripts/codeScan/pylint/pylint.sh | 4 +- .../scripts/codeScan/pyspelling/pyspelling.sh | 6 +- .../scripts/codeScan/scanLog/lpot-bandit.log | 1 - 6 files changed, 157 insertions(+), 200 deletions(-) delete mode 100644 .azure-pipelines/scripts/codeScan/copyright/copy_right.sh delete mode 100644 .azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 4cb89626c89..2440c601459 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,16 +1,15 @@ trigger: none pr: - master + pool: suyue-test variables: - IMAGE_NAME: "code-scan" - IMAGE_TAG: "1.0" + IMAGE_NAME: "neural-compressor" + IMAGE_TAG: "py38" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) - CURRENT_STATUS: true - ER: "1212" stages: - stage: BanditCodeScan @@ -19,7 +18,6 @@ stages: jobs: - job: Bandit displayName: Bandit - condition: eq(variables['CURRENT_STATUS'], 'true') steps: - script: | echo ${BUILD_SOURCESDIRECTORY} @@ -47,164 +45,150 @@ stages: inputs: targetType: "inline" script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh $(CURRENT_STATUS) - echo "------$(CURRENT_STATUS)-------" - echo $(CURRENT_STATUS) + docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh - echo "---$(ER)----$(QR)------" + displayName: "Bandit Check" + - task: PublishPipelineArtifact@1 + condition: always() inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log artifact: bandit publishLocation: "pipeline" + displayName: "PublishPipelineArtifact" + + + - stage: PylintCodeScan + displayName: Pylint Code Scan + dependsOn: [] + jobs: + - job: Pylint + displayName: Pylint + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh + displayName: "Pylint Check" + + - task: PublishPipelineArtifact@1 + condition: always() + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json + artifact: pylint + publishLocation: "pipeline" + displayName: "Status Check" + - # - script: | - # if [ $(CURRENT_STATUS) -ne 0 ] ; then - # exit 1 - # fi - # exit 0 - - - - # - stage: PylintCodeScan - # displayName: Pylint Code Scan - # dependsOn: [] - # jobs: - # - job: Pylint - # displayName: Pylint - # steps: - # - script: | - # if [ $(CURRENT_STATUS) -ne 0 ] ; then - # exit 1 - # fi - # exit 0 - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" - - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh - - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - # artifact: pylint - # publishLocation: "pipeline" - - # - script: | - # if [ $(CURRENT_STATUS) -ne 0 ] ; then - # exit 1 - # fi - # exit 0 - - # - stage: PyspellingCodeScan - # displayName: Pyspelling Code Scan - # dependsOn: [] - # jobs: - # - job: Pyspelling - # displayName: Pyspelling - # steps: - # - script: | - # if [ $(CURRENT_STATUS) -ne 0 ] ; then - # exit 1 - # fi - # exit 0 - # - script: | - # echo ${BUILD_SOURCESDIRECTORY} - # sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - # echo y | docker system prune - # displayName: "Clean workspace" - # - checkout: self - # displayName: "Checkout out Repo" - # - script: | - # if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - # fi - # docker images | grep -i ${IMAGE_NAME} - # if [[ $? -ne 0 ]]; then - # echo "NO Such Image ${IMAGE_NAME}" - # exit 1 - # fi - # displayName: "Build Devel Images" - # - script: | - # docker stop $(docker ps -aq) - # docker rm -vf $(docker ps -aq) || true - # displayName: "Clean Docker" - - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh - - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - # artifact: pyspelling - # publishLocation: "pipeline" - - # - stage: CopyRight - # displayName: CopyRight Code Scan - # dependsOn: [] - # jobs: - # - job: CopyRight - # displayName: CopyRight - # steps: - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # set -ex - # supported_extensions=(py, sh, yaml) - - # git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log - # files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') - - # for file in ${files} - # do - # if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - # echo "Checking license in 1 ${file}" - # if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - # echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - # fi - # else - # echo "Skipping ${file}" - # fi - # done - - # - task: PublishPipelineArtifact@1 - # inputs: - # targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - # artifact: copyright - # publishLocation: "pipeline" - - # - task: Bash@3 - # inputs: - # targetType: "inline" - # script: | - # ls $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - # if [ $? -eq 0 ] ; then - # echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 - # fi - # exit 0 \ No newline at end of file + - stage: PyspellingCodeScan + displayName: Pyspelling Code Scan + dependsOn: [] + jobs: + - job: Pyspelling + displayName: Pyspelling + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: "Clean Docker" + + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh + displayName: "Pyspelling Check" + + - task: PublishPipelineArtifact@1 + condition: always() + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log + artifact: pyspelling + publishLocation: "pipeline" + displayName: "PublishPipelineArtifact" + + - stage: CopyRight + displayName: CopyRight Code Scan + dependsOn: [] + jobs: + - job: CopyRight + displayName: CopyRight + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: "Clean workspace" + - checkout: self + displayName: "Checkout out Repo" + - task: Bash@3 + inputs: + targetType: "inline" + script: | + set -xe + mkdir -p $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + + supported_extensions=(py, sh, yaml) + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log + files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + + for file in ${files} + do + if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then + echo "Checking license in 1 ${file}" + if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then + echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + fi + else + echo "Skipping ${file}" + fi + done + displayName: "CopyRight Check" + + - task: PublishPipelineArtifact@1 + condition: failed() + inputs: + targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + artifact: copyright + publishLocation: "pipeline" + displayName: "PublishPipelineArtifact" \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index e8f868a6f3f..f5b21a45682 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,15 +1,16 @@ -# set -ex +#!/bin/bash +set -ex + +mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog + bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install bandit + python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log exit_code=$? -if [ ${exit_code} -eq 0 ] ; then - sed -i "s|CURRENT_STATUS\: true|CURRENT_STATUS\: false|g" /neural-compressor/azure-pipelines.yml - sed -i "s|ER|QR|g" /neural-compressor/azure-pipelines.yml - - # sed -i 's/CURRENT_STATUS:.*$/CURRENT_STATUS: false' /neural-compressor/azure-pipelines.yml +if [ ${exit_code} -ne 0 ] ; then echo "Bandit exited with non-zero exit code."; exit 1 fi exit 0 diff --git a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh b/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh deleted file mode 100644 index 6187b3ebeb0..00000000000 --- a/.azure-pipelines/scripts/codeScan/copyright/copy_right.sh +++ /dev/null @@ -1,31 +0,0 @@ - -supported_extensions=(py, sh, yaml) - -set -xe -git config --global --add safe.directory /neural_compressor -git --no-pager diff --name-only --no-index $(git show-ref -s remotes/origin/master) /neural_compressor/neural_compressor > /diff.log -# git --no-pager diff --name-only $(git show-ref -s remotes/origin$2) ./$1 > $3/copyright/diff.log - -files=$(cat /diff.log | awk '!a[$0]++') - -for file in ${files} - -do - if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - echo "Checking license in 1 ${file}" - if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - echo ${file} >> /copyright_issue_summary.log - cat /copyright_issue_summary.log - fi - else - echo "Skipping ${file}" - fi -done - - -ls /copyright_issue_summary.log -exit_code=$? -if [ ${exit_code} -e 0 ] ; then - echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 -fi -exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index cef8620dae5..a8b626bcc7f 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -1,4 +1,7 @@ +#!/bin/bash set -ex + +mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install pylint==2.12.1 @@ -15,7 +18,6 @@ pip install -r /neural-compressor/requirements.txt pip install onnxruntime_extensions - python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json exit_code=$? diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 73b431ed8e4..2ff44663c10 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -1,4 +1,7 @@ +#!/bin/bash set -ex + +mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" @@ -8,11 +11,10 @@ pip install -r /neural-compressor/requirements.txt apt-get install aspell -y apt-get install aspell-en -y - - sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml + pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log exit_code=$? if [ ${exit_code} -ne 0 ] ; then diff --git a/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log b/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log deleted file mode 100644 index 9d07aa0df55..00000000000 --- a/.azure-pipelines/scripts/codeScan/scanLog/lpot-bandit.log +++ /dev/null @@ -1 +0,0 @@ -111 \ No newline at end of file From f02df6d1523ec0b57f14e130846d34db85269c5d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Thu, 18 Aug 2022 08:58:24 +0800 Subject: [PATCH 271/296] delete main path yml --- azure-pipelines.yml | 67 --------------------------------------------- 1 file changed, 67 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 77719aa486b..00000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -pr: - - letong_performance - -pool: wenjiao-test - -stages: - - stage: pylint - jobs: - - job: pylint_test - steps: - - task: Bash@3 - inputs: - targetType: "inline" - script: | - pwd - pip install pylint==2.12.1 - pip install intel-tensorflow - python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime ./neural_compressor > /lpot-pylint.json - # # exit_code=$? - # # if [ ${exit_code} -ne 0 ] ; then - # # echo "PyLint exited with non-zero exit code."; exit 1 - # # fi - # # exit 0 - - - stage: bandit - jobs: - - job: bandit_test - steps: - - task: Bash@3 - inputs: - targetType: "inline" -# script: | -# pip install bandit -# # python -m bandit -r -lll -iii neural_compressor > ${WORKSPACE}/lpot-bandit.log -# # exit_code=$? -# # if [ ${exit_code} -ne 0 ] ; then -# # echo "Bandit exited with non-zero exit code."; exit 1 -# # fi -# # exit 0 - - # - stage: pyspelling - # jobs: - # - job: pyspelling_test -# steps: -# - task: Bash@3 -# inputs: -# targetType: "inline" -# script: | -# pip install pyspelling -# # Update paths to validation and lpot repositories -# # VAL_REPO=${WORKSPACE} - -# # sed -i "s|\${VAL_REPO}|$VAL_REPO|g" ${VAL_REPO}/pyspelling_conf.yaml -# # sed -i "s|\${LPOT_REPO}|$REPO_DIR|g" ${VAL_REPO}/pyspelling_conf.yaml -# # echo "Modified config:" -# # cat ${VAL_REPO}/pyspelling_conf.yaml -# # pyspelling -c ${VAL_REPO}/pyspelling_conf.yaml > ${WORKSPACE}/pyspelling_output.log -# # exit_code=$? -# # if [ ${exit_code} -ne 0 ] ; then -# # echo "Pyspelling exited with non-zero exit code."; exit 1 -# # fi -# # exit 0 From 768f96bdda9facdc5c6fda037bff00238e093b64 Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Sun, 21 Aug 2022 18:39:21 +0800 Subject: [PATCH 272/296] update ut pool and bug fix (#24) --- .azure-pipelines/docker/Dockerfile.devel | 3 +- .azure-pipelines/scripts/ut/env_setup.sh | 20 +++++---- .../scripts/ut/run_basic_adaptor.sh | 8 ++-- .../scripts/ut/run_basic_adaptor_tfnewapi.sh | 1 + .azure-pipelines/scripts/ut/run_basic_ipex.sh | 28 +++++++++++++ .../scripts/ut/run_basic_others.sh | 12 ++++-- .azure-pipelines/ut-basic.yml | 42 ++++++++++++++++++- .azure-pipelines/ut-ncoder.yml | 2 +- .azure-pipelines/ut-ux.yml | 2 +- neural_compressor/__init__.py | 1 + 10 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 .azure-pipelines/scripts/ut/run_basic_ipex.sh diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel index 5e9db3f57b7..71ab8bfda6a 100644 --- a/.azure-pipelines/docker/Dockerfile.devel +++ b/.azure-pipelines/docker/Dockerfile.devel @@ -30,7 +30,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ libglib2.0-0 \ numactl \ time \ - wget + wget \ + vim RUN ln -sf $(which python3) /usr/bin/python diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index af008a8bd01..9c101916a1b 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -1,12 +1,13 @@ #!/bin/bash set -x -echo $tensorflow_version -echo $pytorch_version -echo $torchvision_version -echo $onnx_version -echo $onnxruntime_version -echo $mxnet_version +echo "tensorflow version is $tensorflow_version" +echo "pytorch version is $pytorch_version" +echo "torchvision version is $torchvision_version" +echo "ipex version is $ipex_version" +echo "onnx version is $onnx_version" +echo "onnxruntime version is $onnxruntime_version" +echo "mxnet version is $mxnet_version" if [[ "${tensorflow_version}" == *"-official" ]]; then pip install tensorflow==${tensorflow_version%-official} @@ -27,6 +28,11 @@ if [[ "${torchvision_version}" != "" ]]; then pip install torchvision==${torchvision_version} -f https://download.pytorch.org/whl/torch_stable.html fi +if [[ "${ipex_version}" != "" ]]; then + ipex_whl="http://intel-optimized-pytorch.s3.cn-north-1.amazonaws.com.cn/wheels/v1.12.0/intel_extension_for_pytorch-1.12.0%2Bcpu-cp38-cp38-linux_x86_64.whl" + pip install $ipex_whl +fi + if [[ "${onnx_version}" != "" ]]; then pip install onnx==${onnx_version} fi @@ -50,7 +56,7 @@ if [ -f "requirements.txt" ]; then sed -i '/^neural-compressor/d' requirements.txt sed -i '/^intel-tensorflow/d' requirements.txt sed -i '/find-links https:\/\/download.pytorch.org\/whl\/torch_stable.html/d' requirements.txt - sed -i '/^torch/d;/^torchvision/d' requirements.txt + sed -i '/^torch/d;/^torchvision/d;/^intel-extension-for-pytorch/d' requirements.txt sed -i '/^mxnet-mkl/d' requirements.txt sed -i '/^onnx/d;/^onnxruntime/d;/^onnxruntime-extensions/d' requirements.txt n=0 diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index 334be82959e..2a7fdcb283c 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -3,10 +3,10 @@ set -x echo "specify fwk version..." export tensorflow_version='2.9.1' -export pytorch_version='1.11.0+cpu' -export torchvision_version='0.12.0' -export onnx_version='1.9.0' -export onnxruntime_version='1.10.0' +export pytorch_version='1.12.0+cpu' +export torchvision_version='0.13.0+cpu' +export onnx_version='1.11.0' +export onnxruntime_version='1.11.0' export mxnet_version='1.7.0' echo "set up UT env..." diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh index 50b11cdf351..2c9fbf37649 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -5,6 +5,7 @@ echo "run basic adaptor tfnewapi" echo "specify fwk version..." export tensorflow_version='spr-base' +# export FORCE_BF16=1 echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh diff --git a/.azure-pipelines/scripts/ut/run_basic_ipex.sh b/.azure-pipelines/scripts/ut/run_basic_ipex.sh new file mode 100644 index 00000000000..c005e957238 --- /dev/null +++ b/.azure-pipelines/scripts/ut/run_basic_ipex.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -x +python -c "import neural_compressor as nc;print(nc.version.__version__)" +echo "run basic adaptor tfnewapi" + +echo "specify fwk version..." +export pytorch_version='1.12.0+cpu' +export torchvision_version='0.13.0+cpu' +export ipex_version='1.12.0+cpu' + +echo "set up UT env..." +bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh + +cd /neural-compressor/test || exit 1 +find ./ipex -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh + +LOG_DIR=/neural-compressor/log_dir +mkdir -p ${LOG_DIR} +ut_log_name=${LOG_DIR}/ut_ipex.log + +echo "cat run.sh..." +cat run.sh | tee ${ut_log_name} +echo "-------------" +bash run.sh 2>&1 | tee -a ${ut_log_name} + +if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then + exit 1 +fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh index 5cd2e39d1e5..53b61269c7a 100644 --- a/.azure-pipelines/scripts/ut/run_basic_others.sh +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -5,14 +5,17 @@ echo "run basic" echo "specify fwk version..." export tensorflow_version='2.9.1' -export pytorch_version='1.11.0+cpu' -export torchvision_version='0.12.0' -export onnx_version='1.9.0' -export onnxruntime_version='1.10.0' +export pytorch_version='1.12.0+cpu' +export torchvision_version='0.13.0+cpu' +export onnx_version='1.11.0' +export onnxruntime_version='1.11.0' export mxnet_version='1.7.0' echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +echo "copy pre-train model..." +mkdir -p /tmp/.neural_compressor/inc_ut || true +cp -r /tf_dataset/ut-localfile/resnet_v2 /tmp/.neural_compressor/inc_ut || true cd /neural-compressor/test || exit 1 find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh @@ -20,6 +23,7 @@ sed -i '/ adaptor\//d' run.sh sed -i '/ tfnewapi\//d' run.sh sed -i '/ ux\//d' run.sh sed -i '/ neural_coder\//d' run.sh +sed -i '/ ipex\//d' run.sh LOG_DIR=/neural-compressor/log_dir mkdir -p ${LOG_DIR} diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 4e7ec62d2bc..cf19975eb78 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -10,7 +10,7 @@ pr: exclude: - neural_compressor/ux -pool: suyue-test +pool: ICX-16C variables: IMAGE_NAME: 'neural-compressor' @@ -93,6 +93,44 @@ stages: && bash ut/run_basic_adaptor_tfnewapi.sh" displayName: 'Install Neural Compressor and run UT' +- stage: + displayName: Unit Test IPEX + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + + - checkout: self + displayName: "Checkout out Repo" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_ipex.sh" + displayName: 'Install Neural Compressor and run UT' + - stage: displayName: Unit Test other basic case dependsOn: [] @@ -125,7 +163,7 @@ stages: displayName: 'Clean docker' - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc.sh \ && bash ut/run_basic_others.sh" diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml index e2a60d39a76..314a6b4a3c1 100644 --- a/.azure-pipelines/ut-ncoder.yml +++ b/.azure-pipelines/ut-ncoder.yml @@ -8,7 +8,7 @@ pr: include: - neural_coder -pool: suyue-test +pool: ICX-16C variables: IMAGE_NAME: 'neural-compressor' diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 41922165d84..844e8db2a1e 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -8,7 +8,7 @@ pr: include: - neural_compressor/ux -pool: suyue-test +pool: ICX-16C variables: IMAGE_NAME: 'neural-compressor' diff --git a/neural_compressor/__init__.py b/neural_compressor/__init__.py index 06559a3eda5..409737c3741 100644 --- a/neural_compressor/__init__.py +++ b/neural_compressor/__init__.py @@ -14,6 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# for test from .quantization import Quantization from .pruning import Pruning From c9f0461938d8559de02434824edb130d1b73668a Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Wed, 24 Aug 2022 11:20:09 +0800 Subject: [PATCH 273/296] Code scan optimization (#28) --- .azure-pipelines/code-scan.yml | 42 +++++----- .../docker/DockerfileCodeScan.devel | 43 ++++++++++ .azure-pipelines/scripts/change_color.sh | 83 +++++++++++++++++++ .../scripts/codeScan/bandit/bandit.sh | 22 +++-- .../scripts/codeScan/pylint/pylint.sh | 25 +++--- .../scripts/codeScan/pyspelling/lpot_dict.txt | 1 + .../scripts/codeScan/pyspelling/pyspelling.sh | 20 +++-- 7 files changed, 185 insertions(+), 51 deletions(-) create mode 100644 .azure-pipelines/docker/DockerfileCodeScan.devel create mode 100644 .azure-pipelines/scripts/change_color.sh diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 2440c601459..aaa7e328045 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -5,12 +5,13 @@ pr: pool: suyue-test variables: - IMAGE_NAME: "neural-compressor" - IMAGE_TAG: "py38" + IMAGE_NAME: "code-scan" + IMAGE_TAG: "1.0" CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" CURRENT_PATH: $(Build.SourcesDirectory) + stages: - stage: BanditCodeScan displayName: Bandit Code Scan @@ -27,8 +28,8 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/DockerfileCodeScan.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -40,15 +41,12 @@ stages: docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" - - task: Bash@3 inputs: targetType: "inline" script: | docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh - displayName: "Bandit Check" - - task: PublishPipelineArtifact@1 condition: always() inputs: @@ -73,8 +71,8 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/DockerfileCodeScan.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -86,21 +84,19 @@ stages: docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" - - task: Bash@3 inputs: targetType: "inline" script: | docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh displayName: "Pylint Check" - - task: PublishPipelineArtifact@1 condition: always() inputs: targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json artifact: pylint publishLocation: "pipeline" - displayName: "Status Check" + displayName: "PublishPipelineArtifact" - stage: PyspellingCodeScan @@ -118,8 +114,8 @@ stages: - checkout: self displayName: "Checkout out Repo" - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/DockerfileCodeScan.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . fi docker images | grep -i ${IMAGE_NAME} if [[ $? -ne 0 ]]; then @@ -131,14 +127,12 @@ stages: docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true displayName: "Clean Docker" - - task: Bash@3 inputs: targetType: "inline" script: | docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh displayName: "Pyspelling Check" - - task: PublishPipelineArtifact@1 condition: always() inputs: @@ -147,6 +141,7 @@ stages: publishLocation: "pipeline" displayName: "PublishPipelineArtifact" + - stage: CopyRight displayName: CopyRight Code Scan dependsOn: [] @@ -164,8 +159,10 @@ stages: inputs: targetType: "inline" script: | - set -xe - mkdir -p $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + source $(CURRENT_PATH)/.azure-pipelines/scripts/change_color.sh + set -e + mkdir -p $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + RESET="echo -en \\E[0m \\n" # close supported_extensions=(py, sh, yaml) git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log @@ -174,17 +171,18 @@ stages: for file in ${files} do if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then - echo "Checking license in 1 ${file}" if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log - echo "------------------Check for wrong file list !!!!!!!!!!!!!!!!!!!!!!!"; exit 1 + $BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" + cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET; exit 1 fi else - echo "Skipping ${file}" + $LIGHT_PURPLE && echo "Skipping ${file}" && $RESET fi done displayName: "CopyRight Check" - - task: PublishPipelineArtifact@1 condition: failed() inputs: diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.devel new file mode 100644 index 00000000000..93321aa0f14 --- /dev/null +++ b/.azure-pipelines/docker/DockerfileCodeScan.devel @@ -0,0 +1,43 @@ +# +# Copyright (c) 2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG IMAGE_NAME='neural-compressor' +ARG IMAGE_TAG='py38' + +FROM ${IMAGE_NAME}:${IMAGE_TAG} as devel + +# See http://bugs.python.org/issue19846 +ENV LANG C.UTF-8 + + +RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \ + aspell \ + aspell-en + +RUN python -m pip install --no-cache-dir pylint==2.12.1\ + bandit\ + pyspelling\ + google\ + autograd\ + ofa\ + fvcore\ + pymoo\ + onnxruntime_extensions\ + tf_slim\ + transformers\ + horovod\ + flask==2.1.3 + +WORKDIR / diff --git a/.azure-pipelines/scripts/change_color.sh b/.azure-pipelines/scripts/change_color.sh new file mode 100644 index 00000000000..e14b77ee2fa --- /dev/null +++ b/.azure-pipelines/scripts/change_color.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# -------------- general approach start---------------- + +# 1. import this file: + # source path/change_color.sh +# 2. use COLOR/BG: + # $VARIABLE_NAME && out_put_content && $RESET +# 3. COLOR + BG: + # $COLOR/BG_VARIABLE_NAME && $BG/COLOR_VARIABLE_NAME && out_put_content && $RESET +# 4. custom + # abbreviation(change number) + # txt number range (30, 37) + # bg number range (40, 47) + # special effects number range (1, 7) + # echo -en \\E[number1 + ; + number2 + ; + number3 + m" + # e.g - BG_GRAY+LIGHT_RED = "echo -en \\E[47;31m" + +# -------------- general approach end----------------== + + +# general setting +# ------------- light_color start---------------- +# black +LIGHT_BLACK="echo -en \\E[30m" +# red +LIGHT_RED="echo -en \\E[31m" +# green +LIGHT_GREEN="echo -en \\E[32m" +# yellow +LIGHT_YELLOW="echo -en \\E[33m" +# blue +LIGHT_BLUE="echo -en \\E[34m" +# purple +LIGHT_PURPLE="echo -en \\E[35m" +# cyan +LIGHT_CYAN="echo -en \\E[36m" +# gray +LIGHT_GRAY="echo -en \\E[37m" +# ------------- light_color end---------------- + +# ------------- bold_color start---------------- +# black +BOLD_BLACK="echo -en \\E[1;30m" +# red +BOLD_RED="echo -en \\E[1;31m" +# green +BOLD_GREEN="echo -en \\E[1;32m" +# yellow +BOLD_YELLOW="echo -en \\E[1;33m" +# blue +BOLD_BLUE="echo -en \\E[1;34m" +# purple +BOLD_PURPLE="echo -en \\E[1;35m" +# cyan +BOLD_CYAN="echo -en \\E[1;36m" +# gray +BOLD_GRAY="echo -en \\E[1;37m" +# ------------- bold_color end---------------- + +# ------------- background_color start---------------- +# black +BG_BLACK="echo -en \\E[40m" +# red +BG_RED="echo -en \\E[41m" +# green +BG_GREEN="echo -en \\E[42m" +# yellow +BG_YELLOW="echo -en \\E[43m" +# blue +BG_BLUE="echo -en \\E[44m" +# purple +BG_PURPLE="echo -en \\E[45m" +# cyan +BG_CYAN="echo -en \\E[46m" +# gray +BG_GRAY="echo -en \\E[47m" +# ------------- background_color end---------------- + +# close +RESET="echo -en \\E[0m" + + diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index f5b21a45682..bb4f8633e83 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,17 +1,29 @@ #!/bin/bash -set -ex +set -e mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog - bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" -pip install bandit +# error +RED="echo -en \\E[1;31m" +# succeed +PURPLE="echo -en \\E[1;35m" +# succeed hint +LIGHT_PURPLE="echo -en \\E[35m" +# log output hint +YELLOW="echo -en \\E[1;33m" +# close +RESET="echo -en \\E[0m \\n" python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log +$YELLOW && echo " ----------------- Current log file output start --------------------------" +cat $bandit_log_dir/lpot-bandit.log +$YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + exit_code=$? if [ ${exit_code} -ne 0 ] ; then - echo "Bandit exited with non-zero exit code."; exit 1 + $RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details. " && $RESET; exit 1 fi -exit 0 +$PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index a8b626bcc7f..4fd5dde8d14 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -1,27 +1,22 @@ #!/bin/bash -set -ex +set -e +source /neural-compressor/.azure-pipelines/scripts/change_color.sh mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" -pip install pylint==2.12.1 -pip install flask==2.1.3 -pip install google -pip install horovod -pip install ofa -pip install fvcore -pip install autograd -pip install pymoo -pip install tf_slim -pip install transformers pip install -r /neural-compressor/requirements.txt -pip install onnxruntime_extensions - python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json +# code-scan close +RESET="echo -en \\E[0m \\n" + +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" +cat $pylint_log_dir/lpot-pylint.json +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET exit_code=$? if [ ${exit_code} -ne 0 ] ; then - echo "PyLint exited with non-zero exit code."; exit 1 + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET; exit 1 fi -exit 0 \ No newline at end of file +$BOLD_PURPLE && echo "Congratulations, Pylint check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 \ No newline at end of file diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt index 25bf61a1cef..88f398f09c7 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -95,6 +95,7 @@ autgrad autogenerate autograd AutoMixPrecision +AutoQuant autopep Autoregressive ava diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 2ff44663c10..72bcd763155 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -1,25 +1,27 @@ #!/bin/bash -set -ex +set -e +source /neural-compressor/.azure-pipelines/scripts/change_color.sh mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog pyspelling_dir="/neural-compressor/.azure-pipelines/scripts/codeScan" pyspelling_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" -pip install pyspelling pip install -r /neural-compressor/requirements.txt -apt-get install aspell -y -apt-get install aspell-en -y - sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml - pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log +# code-scan close +RESET="echo -en \\E[0m \\n" + +$BOLD_YELLOW && echo "------------------- Current log file output start --------------------------" +cat $pyspelling_log_dir/lpot_pyspelling.log +$BOLD_YELLOW && echo "------------------- Current log file output end ----------------------------" && $RESET + exit_code=$? if [ ${exit_code} -ne 0 ] ; then - echo "Pyspelling exited with non-zero exit code."; exit 1 + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pyspelling error details." && $RESET; exit 1 fi -exit 0 - +$BOLD_PURPLE && echo "Congratulations, Pyspelling check passed!" && $LIGHT_PURPLE && echo "You can click on the artifact button to see the log details." && $RESET; exit 0 From 91fa1a5025ef56acdd95d8226c771389f4016d87 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Fri, 2 Sep 2022 17:29:14 +0800 Subject: [PATCH 274/296] Code scan modify (#36) --- .azure-pipelines/code-scan.yml | 2 +- .../scripts/codeScan/bandit/bandit.sh | 28 +++++++------------ .../scripts/codeScan/pylint/pylint.sh | 5 ++-- .../scripts/codeScan/pyspelling/pyspelling.sh | 6 ++-- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index aaa7e328045..d44ee11adae 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -2,7 +2,7 @@ trigger: none pr: - master -pool: suyue-test +pool: ICX-16C variables: IMAGE_NAME: "code-scan" diff --git a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh index bb4f8633e83..a23f2f3000d 100644 --- a/.azure-pipelines/scripts/codeScan/bandit/bandit.sh +++ b/.azure-pipelines/scripts/codeScan/bandit/bandit.sh @@ -1,29 +1,21 @@ #!/bin/bash -set -e +source /neural-compressor/.azure-pipelines/scripts/change_color.sh mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog bandit_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" -# error -RED="echo -en \\E[1;31m" -# succeed -PURPLE="echo -en \\E[1;35m" -# succeed hint -LIGHT_PURPLE="echo -en \\E[35m" -# log output hint -YELLOW="echo -en \\E[1;33m" -# close -RESET="echo -en \\E[0m \\n" +python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log +exit_code=$? -python -m bandit -r -lll -iii /neural-compressor/neural_compressor > $bandit_log_dir/lpot-bandit.log +# code-scan close +RESET="echo -en \\E[0m \\n" -$YELLOW && echo " ----------------- Current log file output start --------------------------" +$BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" cat $bandit_log_dir/lpot-bandit.log -$YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET +$BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET + -exit_code=$? if [ ${exit_code} -ne 0 ] ; then - $RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details. " && $RESET; exit 1 + $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Bandit error details." && $RESET; exit 1 fi -$PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 - +$BOLD_PURPLE && echo "Congratulations, Bandit check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET; exit 0 diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 4fd5dde8d14..9e8a02d33be 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -e source /neural-compressor/.azure-pipelines/scripts/change_color.sh mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog @@ -8,6 +7,8 @@ pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install -r /neural-compressor/requirements.txt python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json +exit_code=$? + # code-scan close RESET="echo -en \\E[0m \\n" @@ -15,7 +16,7 @@ $BOLD_YELLOW && echo " ----------------- Current log file output start -------- cat $pylint_log_dir/lpot-pylint.json $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET -exit_code=$? + if [ ${exit_code} -ne 0 ] ; then $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET; exit 1 fi diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh index 72bcd763155..5a472fae374 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh +++ b/.azure-pipelines/scripts/codeScan/pyspelling/pyspelling.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e + source /neural-compressor/.azure-pipelines/scripts/change_color.sh mkdir -p /neural-compressor/.azure-pipelines/scripts/codeScan/scanLog @@ -12,6 +12,8 @@ sed -i "s|\${VAL_REPO}|$pyspelling_dir|g" $pyspelling_dir/pyspelling/pyspelling_ sed -i "s|\${LPOT_REPO}|/neural-compressor|g" $pyspelling_dir/pyspelling/pyspelling_conf.yaml pyspelling -c $pyspelling_dir/pyspelling/pyspelling_conf.yaml > $pyspelling_log_dir/lpot_pyspelling.log +exit_code=$? + # code-scan close RESET="echo -en \\E[0m \\n" @@ -19,7 +21,7 @@ $BOLD_YELLOW && echo "------------------- Current log file output start ------- cat $pyspelling_log_dir/lpot_pyspelling.log $BOLD_YELLOW && echo "------------------- Current log file output end ----------------------------" && $RESET -exit_code=$? + if [ ${exit_code} -ne 0 ] ; then $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pyspelling error details." && $RESET; exit 1 fi From d7620ac82ee21ee153d40213e3a5536e0f5efab7 Mon Sep 17 00:00:00 2001 From: Letong Han <106566639+letonghan@users.noreply.github.com> Date: Fri, 2 Sep 2022 22:32:28 +0800 Subject: [PATCH 275/296] Add model test scripts for Azure CI (#37) --- .azure-pipelines/model-test.yml | 319 ++++++++++ .../scripts/models/collect_log_all.py | 41 ++ .../scripts/models/collect_log_model.py | 116 ++++ .../scripts/models/generate_report.sh | 578 ++++++++++++++++++ .../scripts/models/model_template.yml | 45 ++ .../scripts/models/run_benchmark_common.sh | 96 +++ .../models/run_model_trigger_common.sh | 153 +++++ .../models/run_mxnet_models_trigger.sh | 41 ++ .../models/run_onnxrt_models_trigger.sh | 61 ++ .../models/run_pytorch_models_trigger.sh | 53 ++ .../models/run_tensorflow_models_trigger.sh | 111 ++++ .../scripts/models/run_tuning_common.sh | 51 ++ .../scripts/models/update_yaml_config.py | 332 ++++++++++ 13 files changed, 1997 insertions(+) create mode 100644 .azure-pipelines/model-test.yml create mode 100644 .azure-pipelines/scripts/models/collect_log_all.py create mode 100644 .azure-pipelines/scripts/models/collect_log_model.py create mode 100644 .azure-pipelines/scripts/models/generate_report.sh create mode 100644 .azure-pipelines/scripts/models/model_template.yml create mode 100644 .azure-pipelines/scripts/models/run_benchmark_common.sh create mode 100644 .azure-pipelines/scripts/models/run_model_trigger_common.sh create mode 100644 .azure-pipelines/scripts/models/run_mxnet_models_trigger.sh create mode 100644 .azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh create mode 100644 .azure-pipelines/scripts/models/run_pytorch_models_trigger.sh create mode 100644 .azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh create mode 100644 .azure-pipelines/scripts/models/run_tuning_common.sh create mode 100644 .azure-pipelines/scripts/models/update_yaml_config.py diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml new file mode 100644 index 00000000000..5755d93d8a2 --- /dev/null +++ b/.azure-pipelines/model-test.yml @@ -0,0 +1,319 @@ + +trigger: none +pr: + - master + +pool: ICX-16C + +variables: + OUT_SCRIPT_PATH: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models + SCRIPT_PATH: /neural-compressor/.azure-pipelines/scripts + MODEL_LOG_DIR: /neural-compressor/.azure-pipelines + IMAGE_NAME: 'neural-compressor' + IMAGE_TAG: 'py38' + +parameters: +- name: test_tf + displayName: Run tensorflow models? + type: boolean + default: true +- name: test_pt + displayName: Run pytorch models? + type: boolean + default: true +- name: test_onnx + displayName: Run onnxrt models? + type: boolean + default: true +- name: test_mxnet + displayName: Run mxnet models? + type: boolean + default: true + +stages: +- ${{ if eq(parameters.test_tf, true) }}: + - stage: TensorflowModels + displayName: Tensorflow Model Test + dependsOn: [] + jobs: + - job: resnet50v1_5 + displayName: resnet50v1.5 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet50v1.5 + framework: tensorflow + - script: | + if [ $(tensorflow_resnet50v1.5_failed) == 'true' ]; then + echo "[Failed] Model resnet50v1.5 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: ssd_resnet50_v1 + displayName: ssd_resnet50_v1 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: ssd_resnet50_v1 + framework: tensorflow + - script: | + if [ $(tensorflow_ssd_resnet50_v1_failed) == 'true' ]; then + echo "[Failed] Model ssd_resnet50_v1 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: ssd_mobilenet_v1_ckpt + displayName: ssd_mobilenet_v1_ckpt + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: ssd_mobilenet_v1_ckpt + framework: tensorflow + - script: | + if [ $(tensorflow_ssd_mobilenet_v1_ckpt_failed) == 'true' ]; then + echo "[Failed] Model ssd_mobilenet_v1_ckpt failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: inception_v1 + displayName: inception_v1 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: inception_v1 + framework: tensorflow + - script: | + if [ $(tensorflow_inception_v1_failed) == 'true' ]; then + echo "[Failed] Model inception_v1 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: resnet50_fashion + displayName: resnet50_fashion + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet50_fashion + framework: tensorflow + - script: | + if [ $(tensorflow_resnet50_fashion_failed) == 'true' ]; then + echo "[Failed] Model resnet50_fashion failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: darknet19 + displayName: darknet19 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: darknet19 + framework: tensorflow + - script: | + if [ $(tensorflow_darknet19_failed) == 'true' ]; then + echo "[Failed] Model darknet19 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: densenet121 + displayName: densenet-121 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: densenet-121 + framework: tensorflow + - script: | + if [ $(tensorflow_densenet-121_failed) == 'true' ]; then + echo "[Failed] Model densenet-121 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: resnet101 + displayName: resnet-101 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet-101 + framework: tensorflow + - script: | + if [ $(tensorflow_resnet-101_failed) == 'true' ]; then + echo "[Failed] Model resnet-101 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + +- ${{ if eq(parameters.test_pt, true) }}: + - stage: PyTorchModels + displayName: PyTorch Model Test + dependsOn: [] + jobs: + - job: resnet18 + displayName: resnet18 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet18 + framework: pytorch + - script: | + if [ $(pytorch_resnet18_failed) == 'true' ]; then + echo "[Failed] Model resnet18 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: resnet18_fx + displayName: resnet18_fx + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet18_fx + framework: pytorch + - script: | + if [ $(pytorch_resnet18_fx_failed) == 'true' ]; then + echo "[Failed] Model resnet18_fx failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + +- ${{ if eq(parameters.test_mxnet, true) }}: + - stage: MXNetModels + displayName: Mxnet Model Test + dependsOn: [] + jobs: + - job: resnet50v1 + displayName: resnet50v1 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet50v1 + framework: mxnet + - script: | + if [ $(mxnet_resnet50v1_failed) == 'true' ]; then + echo "[Failed] Model resnet50v1 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + +- ${{ if eq(parameters.test_onnx, true) }}: + - stage: ONNXModels + displayName: ONNXrt Model Test + dependsOn: [] + jobs: + - job: resnet50_v1_12 + displayName: resnet50-v1-12 + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: resnet50-v1-12 + framework: onnxrt + - script: | + if [ $(onnxrt_resnet50-v1-12_failed) == 'true' ]; then + echo "[Failed] Model resnet50-v1-12 failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: bert_base_MRPC_static + displayName: bert_base_MRPC_static + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: bert_base_MRPC_static + framework: onnxrt + - script: | + if [ $(onnxrt_bert_base_MRPC_static_failed) == 'true' ]; then + echo "[Failed] Model bert_base_MRPC_static failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - job: bert_base_MRPC_dynamic + displayName: bert_base_MRPC_dynamic + steps: + - template: scripts/models/model_template.yml + parameters: + modelName: bert_base_MRPC_dynamic + framework: onnxrt + - script: | + if [ $(onnxrt_bert_base_MRPC_dynamic_failed) == 'true' ]; then + echo "[Failed] Model bert_base_MRPC_dynamic failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" + + - stage: GenerateLogs + displayName: Generate Report + dependsOn: [ TensorflowModels, PyTorchModels, MXNetModels, ONNXModels ] + condition: succeededOrFailed() + jobs: + - job: GenerateReport + displayName: generate report + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: 'Clean workspace' + - checkout: self + clean: true + displayName: 'Checkout out Repo' + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + patterns: '**/*_summary.log' + path: $(OUT_SCRIPT_PATH) + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + patterns: '**/*_tuning_info.log' + path: $(OUT_SCRIPT_PATH) + - task: UsePythonVersion@0 + displayName: 'Use Python 3.8.0' + inputs: + versionSpec: '3.8.0' + - script: | + echo "------ Collecting logs ------" + cd ${OUT_SCRIPT_PATH} + mkdir generated + mkdir last_generated + python -u collect_log_all.py --logs_dir $(OUT_SCRIPT_PATH) --output_dir generated + displayName: "Collect all logs" + - task: DownloadPipelineArtifact@2 + inputs: + source: 'specific' + artifact: 'FinalReport' + patterns: '**.log' + path: $(OUT_SCRIPT_PATH)/last_generated + project: 'test-lpot-public' + pipeline: 45 + runVersion: 'specific' + runId: $(model_runID) + retryDownloadCount: 3 + displayName: 'Download last logs' + - script: | + echo "------ Generating final report.html ------" + cd ${OUT_SCRIPT_PATH} + /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated + displayName: 'Generate report' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(OUT_SCRIPT_PATH)/generated + artifact: FinalReport + publishLocation: 'pipeline' + displayName: "Publish report" + - script: | + if [ $(is_perf_reg) == 'true' ]; then + echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports." + exit 1 + fi + displayName: "Specify performance regression" + + + + diff --git a/.azure-pipelines/scripts/models/collect_log_all.py b/.azure-pipelines/scripts/models/collect_log_all.py new file mode 100644 index 00000000000..61fe9454c2e --- /dev/null +++ b/.azure-pipelines/scripts/models/collect_log_all.py @@ -0,0 +1,41 @@ +import re +import os +import platform +import argparse + +parser = argparse.ArgumentParser(allow_abbrev=False) +parser.add_argument("--logs_dir", type=str, default=".") +parser.add_argument("--output_dir", type=str, default=".") +args = parser.parse_args() +print(args) + + +def main(): + file_dir = args.logs_dir + summary_content = ['OS;Platform;Framework;Version;Precision;Model;Mode;Type;BS;Value;Url\n'] + tuning_info_content = ['OS;Platform;Framework;Version;Model;Strategy;Tune_time\n'] + # get full path of all files + for root, dirs, files in os.walk(file_dir): + for name in files: + file_name = os.path.join(root, name) + print(file_name) + if '_summary.log' in name: + for line in open(file_name, "r"): + # print(line) + if 'linux' in line: + summary_content.append(line) + if '_tuning_info.log' in name: + for line in open(file_name, "r"): + # print(line) + if 'linux' in line: + tuning_info_content.append(line) + f = open(args.output_dir + '/summary.log', "a") + for summary in summary_content: + f.writelines(str(summary)) + f2 = open(args.output_dir + '/tuning_info.log', "a") + for tuning_info in tuning_info_content: + f2.writelines(str(tuning_info)) + + +if __name__ == '__main__': + main() diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py new file mode 100644 index 00000000000..10038ed668a --- /dev/null +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -0,0 +1,116 @@ +import re +import os +import platform +import argparse + + +parser = argparse.ArgumentParser(allow_abbrev=False) +parser.add_argument("--framework", type=str, required=True) +parser.add_argument("--fwk_ver", type=str, required=True) +parser.add_argument("--model", type=str, required=True) +parser.add_argument("--logs_dir", type=str, default=".") +parser.add_argument("--output_dir", type=str, default=".") +parser.add_argument("--build_id", type=str, default="3117") +args = parser.parse_args() +print('===== collecting log model =======') +print('build_id: '+args.build_id) +OS='linux' +PLATFORM='icx' +URL ='https://dev.azure.com/lpot-inc/test-lpot-public/_build/results?buildId='+args.build_id+'&view=artifacts&pathAsName=false&type=publishedArtifacts' + +print(args) + + +def main(): + results = [] + tuning_infos = [] + tuning_log = os.path.join(args.logs_dir, f"{args.framework}-{args.model}-tune.log") + print("tuning log dir is {}".format(tuning_log)) + # get model tuning results + if os.path.exists(tuning_log): + print('tuning log found') + tmp = {'fp32_acc': 0, 'int8_acc': 0, 'tuning_trials': 0} + with open(tuning_log, "r") as f: + for line in f: + parse_tuning_line(line, tmp) + print(tmp) + # set model status failed + if tmp['fp32_acc']==0 or tmp['int8_acc']==0: + os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') + results.append('{};{};{};{};FP32;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['fp32_acc'], URL)) + results.append('{};{};{};{};INT8;{};Inference;Accuracy;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['int8_acc'], URL)) + tuning_infos.append(';'.join([OS, PLATFORM, args.framework, args.fwk_ver, args.model, tmp['strategy'], str(tmp['tune_time']), str(tmp['tuning_trials']), URL, f"{round(tmp['max_mem_size'] / tmp['total_mem_size'] * 100, 4)}%"])+'\n') + # get model benchmark results + for precision in ['int8', 'fp32']: + throughput = 0.0 + for root, dirs, files in os.walk(args.logs_dir): + for name in files: + file_name = os.path.join(root, name) + print(file_name) + if 'performance-'+precision in name: + for line in open(file_name, "r"): + result = parse_perf_line(line) + if result: + throughput += result + # set model status failed + if throughput==0.0: + os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') + results.append('{};{};{};{};{};{};Inference;Performance;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, precision.upper(), args.model, throughput, URL)) + # write model logs + f = open(args.output_dir+'/'+args.framework+'_'+args.model+'_summary.log', "a") + f.writelines("OS;Platform;Framework;Version;Precision;Model;Mode;Type;BS;Value;Url\n") + for result in results: + f.writelines(str(result)) + f2 = open(args.output_dir + '/'+args.framework+'_'+args.model+'_tuning_info.log', "a") + f2.writelines("OS;Platform;Framework;Version;Model;Strategy;Tune_time\n") + for tuning_info in tuning_infos: + f2.writelines(str(tuning_info)) + + +def parse_tuning_line(line, tmp): + tuning_strategy = re.search(r"Tuning strategy:\s+([A-Za-z]+)", line) + if tuning_strategy and tuning_strategy.group(1): + tmp['strategy'] = tuning_strategy.group(1) + + baseline_acc = re.search(r"FP32 baseline is:\s+\[Accuracy:\s(\d+(\.\d+)?), Duration \(seconds\):\s*(\d+(\.\d+)?)\]", + line) + if baseline_acc and baseline_acc.group(1): + tmp['fp32_acc'] = float(baseline_acc.group(1)) + + tuned_acc = re.search(r"Best tune result is:\s+\[Accuracy:\s(\d+(\.\d+)?), Duration \(seconds\):\s(\d+(\.\d+)?)\]", line) + if tuned_acc and tuned_acc.group(1): + tmp['int8_acc'] = float(tuned_acc.group(1)) + + tune_trial = re.search(r"Tune \d*\s*result is:", line) + if tune_trial: + tmp['tuning_trials'] += 1 + + tune_time = re.search(r"Tuning time spend:\s+(\d+(\.\d+)?)s", line) + if tune_time and tune_time.group(1): + tmp['tune_time'] = int(tune_time.group(1)) + + fp32_model_size = re.search(r"The input model size is:\s+(\d+(\.\d+)?)", line) + if fp32_model_size and fp32_model_size.group(1): + tmp['fp32_model_size'] = int(fp32_model_size.group(1)) + + int8_model_size = re.search(r"The output model size is:\s+(\d+(\.\d+)?)", line) + if int8_model_size and int8_model_size.group(1): + tmp['int8_model_size'] = int(int8_model_size.group(1)) + + total_mem_size = re.search(r"Total resident size\D*([0-9]+)", line) + if total_mem_size and total_mem_size.group(1): + tmp['total_mem_size'] = float(total_mem_size.group(1)) + + max_mem_size = re.search(r"Maximum resident set size\D*([0-9]+)", line) + if max_mem_size and max_mem_size.group(1): + tmp['max_mem_size'] = float(max_mem_size.group(1)) + + +def parse_perf_line(line) -> float: + throughput = re.search(r"Throughput:\s+(\d+(\.\d+)?)", line) + if throughput and throughput.group(1): + return float(throughput.group(1)) + + +if __name__ == '__main__': + main() diff --git a/.azure-pipelines/scripts/models/generate_report.sh b/.azure-pipelines/scripts/models/generate_report.sh new file mode 100644 index 00000000000..c5cbb6080f5 --- /dev/null +++ b/.azure-pipelines/scripts/models/generate_report.sh @@ -0,0 +1,578 @@ +#!/bin/bash + +# WORKSPACE=. +# summaryLog=summary.log +# summaryLogLast=summary.log +# tuneLog=tuning_info.log +# tuneLogLast=tuning_info.log +# overview_log=summary_overview.log +# coverage_summary=coverage_summary.log +# nc_code_lines_summary=nc_code_lines_summary.csv +# engine_code_lines_summary=engine_code_lines_summary.csv + +#lines_coverage_threshold=80 +#branches_coverage_threshold=75 +# +#pass_status="Pass" +#fail_status="Fail" +#verify_status="Verify" + + +# shellcheck disable=SC2120 + +while [[ $# -gt 0 ]];do + key=${1} + case ${key} in + -w|--WORKSPACE) + WORKSPACE=${2} + shift 2 + ;; + --script_path) + script_path=${2} + shift 2 + ;; + --output_dir) + output_dir=${2} + shift 2 + ;; + --last_logt_dir) + last_logt_dir=${2} + shift 2 + ;; + *) + shift + ;; + esac +done + +echo "workspace: ${WORKSPACE}" +echo "script_path: ${script_path}" + +summaryLog="${WORKSPACE}/summary.log" +tuneLog="${WORKSPACE}/tuning_info.log" +echo "summaryLog: ${summaryLog}" +echo "tuneLog: ${tuneLog}" + +echo "last_logt_dir: ${last_logt_dir}" +summaryLogLast="${last_logt_dir}/summary.log" +tuneLogLast="${last_logt_dir}/tuning_info.log" +echo "summaryLogLast: ${summaryLogLast}" +echo "tuneLogLast: ${tuneLogLast}" + + + +function main { + generate_html_head + generate_html_body + generate_results + generate_html_footer + +} + +function generate_inference { +# echo "Generating inference" + awk -v framework="${framework}" -v fw_version="${fw_version}" -v model="${model}" -v os="${os}" -v platform=${platform} -F ';' ' + BEGINE { + fp32_perf_bs = "nan"; + fp32_perf_value = "nan"; + fp32_perf_url = "nan"; + fp32_acc_bs = "nan"; + fp32_acc_value = "nan"; + fp32_acc_url = "nan"; + + int8_perf_bs = "nan"; + int8_perf_value = "nan"; + int8_perf_url = "nan"; + int8_acc_bs = "nan"; + int8_acc_value = "nan"; + int8_acc_url = "nan"; + }{ + if($1 == os && $2 == platform && $3 == framework && $4 == fw_version && $6 == model) { + // FP32 + if($5 == "FP32") { + // Performance + if($8 == "Performance") { + fp32_perf_bs = $9; + fp32_perf_value = $10; + fp32_perf_url = $11; + } + // Accuracy + if($8 == "Accuracy") { + fp32_acc_bs = $9; + fp32_acc_value = $10; + fp32_acc_url = $11; + } + } + + // INT8 + if($5 == "INT8") { + // Performance + if($8 == "Performance") { + int8_perf_bs = $9; + int8_perf_value = $10; + int8_perf_url = $11; + } + // Accuracy + if($8 == "Accuracy") { + int8_acc_bs = $9; + int8_acc_value = $10; + int8_acc_url = $11; + } + } + } + }END { + printf("%s;%s;%s;%s;", int8_perf_bs,int8_perf_value,int8_acc_bs,int8_acc_value); + printf("%s;%s;%s;%s;", fp32_perf_bs,fp32_perf_value,fp32_acc_bs,fp32_acc_value); + printf("%s;%s;%s;%s;", int8_perf_url,int8_acc_url,fp32_perf_url,fp32_acc_url); + } + ' "$1" +} + +function generate_html_core { + echo "--- current values ---" + echo ${current_values} + echo "--- last values ---" + echo ${last_values} + tuning_strategy=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $6}') + tuning_time=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $7}') + tuning_count=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $8}') + tuning_log=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLog} |awk -F';' '{print $9}') + echo "${platform}${os}${framework}${fw_version}${model}New${tuning_strategy}" >> ${output_dir}/report.html + echo "${tuning_time}${tuning_count}" >> ${output_dir}/report.html + + tuning_strategy=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $6}') + tuning_time=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $7}') + tuning_count=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $8}') + tuning_log=$(grep "^${os};${platform};${framework};${fw_version};${model};" ${tuneLogLast} |awk -F';' '{print $9}') + + echo |awk -F ';' -v current_values="${current_values}" -v last_values="${last_values}" \ + -v tuning_strategy="${tuning_strategy}" -v tuning_time="${tuning_time}" \ + -v tuning_count="${tuning_count}" -v tuning_log="${tuning_log}" -F ';' ' + + function abs(x) { return x < 0 ? -x : x } + + function show_new_last(batch, link, value, metric) { + if(value ~/[1-9]/) { + if (metric == "perf") { + printf("%s %.2f\n",batch,link,value); + } else { + printf("%s %.2f%\n",batch,link,value*100); + } + } else { + if(link == "" || value == "N/A") { + printf(" \n"); + } else { + printf("%s Failure\n",batch,link); + } + } + } + + function compare_current(int8_result, fp32_result, metric) { + + if(int8_result ~/[1-9]/ && fp32_result ~/[1-9]/) { + if(metric == "acc") { + target = (int8_result - fp32_result) / fp32_result; + if(target >= -0.01) { + printf("%.2f %", target*100); + }else if(target < -0.05) { + printf("%.2f %", target*100); + job_status = "fail" + }else{ + printf("%.2f %", target*100); + } + }else if(metric == "perf") { + target = int8_result / fp32_result; + if(target >= 1.5) { + printf("%.2f", target); + }else if(target < 1) { + printf("%.2f", target); + job_status = "fail" + }else{ + printf("%.2f", target); + } + } + else { + target = int8_result / fp32_result; + if(target >= 2) { + printf("%.2f", target); + }else if(target < 1) { + printf("%.2f", target); + job_status = "fail" + }else{ + printf("%.2f", target); + } + } + }else { + printf(""); + } + } + + function compare_result(new_result, previous_result, metric) { + + if (new_result ~/[1-9]/ && previous_result ~/[1-9]/) { + if(metric == "acc") { + target = new_result - previous_result; + if(target > -0.00001 && target < 0.00001) { + status_png = "background-color:#90EE90"; + } else { + status_png = "background-color:#FFD2D2"; + job_status = "fail" + } + printf("%.2f %", status_png, target*100); + } else { + target = new_result / previous_result; + if(target >= 0.945) { + status_png = "background-color:#90EE90"; + } else { + status_png = "background-color:#FFD2D2"; + job_status = "fail" + } + printf("%.2f", status_png, target); + } + } else { + if(new_result == nan && previous_result == nan){ + printf(""); + } else{ + if(new_result == nan) { + job_status = "fail" + status_png = "background-color:#FFD2D2"; + printf("", status_png); + } else{ + printf(""); + } + } + } + } + + BEGIN { + job_status = "pass" + // issue list + jira_mobilenet = "https://jira01.devtools.intel.com/browse/PADDLEQ-384"; + jira_resnext = "https://jira01.devtools.intel.com/browse/PADDLEQ-387"; + jira_ssdmobilenet = "https://jira01.devtools.intel.com/browse/PADDLEQ-406"; + }{ + // Current values + split(current_values,current_value,";"); + + // Current + + // INT8 Performance results + int8_perf_batch=current_value[1] + int8_perf_value=current_value[2] + int8_perf_url=current_value[9] + show_new_last(int8_perf_batch, int8_perf_url, int8_perf_value, "perf"); + + // INT8 Accuracy results + int8_acc_batch=current_value[3] + int8_acc_value=current_value[4] + int8_acc_url=current_value[10] + show_new_last(int8_acc_batch, int8_acc_url, int8_acc_value, "acc"); + + // FP32 Performance results + fp32_perf_batch=current_value[5] + fp32_perf_value=current_value[6] + fp32_perf_url=current_value[11] + show_new_last(fp32_perf_batch, fp32_perf_url, fp32_perf_value, "perf"); + + // FP32 Accuracy results + fp32_acc_batch=current_value[7] + fp32_acc_value=current_value[8] + fp32_acc_url=current_value[12] + show_new_last(fp32_acc_batch, fp32_acc_url, fp32_acc_value, "acc"); + + // Compare Current + + compare_current(int8_perf_value, fp32_perf_value, "perf"); + compare_current(int8_acc_value, fp32_acc_value, "acc"); + + // Last values + split(last_values,last_value,";"); + + // Last + printf("\nLast%1$s%2$s%3$s", tuning_strategy, tuning_time, tuning_count, tuning_log); + + // Show last INT8 Performance results + last_int8_perf_batch=last_value[1] + last_int8_perf_value=last_value[2] + last_int8_perf_url=last_value[9] + show_new_last(last_int8_perf_batch, last_int8_perf_url, last_int8_perf_value, "perf"); + + // Show last INT8 Accuracy results + last_int8_acc_batch=last_value[3] + last_int8_acc_value=last_value[4] + last_int8_acc_url=last_value[10] + show_new_last(last_int8_acc_batch, last_int8_acc_url, last_int8_acc_value, "acc"); + + // Show last FP32 Performance results + last_fp32_perf_batch=last_value[5] + last_fp32_perf_value=last_value[6] + last_fp32_perf_url=last_value[11] + show_new_last(last_fp32_perf_batch, last_fp32_perf_url, last_fp32_perf_value, "perf"); + + // Show last FP32 Accuracy results + last_fp32_acc_batch=last_value[7] + last_fp32_acc_value=last_value[8] + last_fp32_acc_url=last_value[12] + show_new_last(last_fp32_acc_batch, last_fp32_acc_url, last_fp32_acc_value, "acc"); + + printf("") + + // current vs last + printf("\nNew/Last"); + + // Compare INT8 Performance results + compare_result(int8_perf_value, last_int8_perf_value,"perf"); + + // Compare INT8 Accuracy results + compare_result(int8_acc_value, last_int8_acc_value, "acc"); + + // Compare FP32 Performance results + compare_result(fp32_perf_value, last_fp32_perf_value, "perf"); + + // Compare INT8 Performance results + compare_result(fp32_acc_value, last_fp32_acc_value, "acc"); + + printf("\n"); + + } END{ + printf("\n%s", job_status); + } + ' >> ${output_dir}/report.html + job_state=$(tail -1 ${WORKSPACE}/report.html) + sed -i '$s/.*//' ${WORKSPACE}/report.html + + if [ ${job_state} == 'fail' ]; then + echo "====== perf_reg ======" + echo "##vso[task.setvariable variable=is_perf_reg]true" + fi +} + +function generate_results { + echo "Generating tuning results" + oses=$(sed '1d' ${summaryLog} |cut -d';' -f1 | awk '!a[$0]++') + echo ${oses} + + for os in ${oses[@]} + do + platforms=$(sed '1d' ${summaryLog} |grep "^${os}" |cut -d';' -f2 | awk '!a[$0]++') + echo ${platforms} + for platform in ${platforms[@]} + do + frameworks=$(sed '1d' ${summaryLog} |grep "^${os};${platform}" |cut -d';' -f3 | awk '!a[$0]++') + echo ${frameworks} + for framework in ${frameworks[@]} + do + fw_versions=$(sed '1d' ${summaryLog} |grep "^${os};${platform};${framework}" |cut -d';' -f4 | awk '!a[$0]++') + echo ${fw_versions} + for fw_version in ${fw_versions[@]} + do + models=$(sed '1d' ${summaryLog} |grep "^${os};${platform};${framework};${fw_version}" |cut -d';' -f6 | awk '!a[$0]++') + echo ${models} + for model in ${models[@]} + do + echo "--- processing model ---" + echo ${model} + current_values=$(generate_inference ${summaryLog}) + echo "| current value |" + echo ${current_values} + last_values=$(generate_inference ${summaryLogLast}) + echo "| last value |" + echo ${last_values} + + generate_html_core ${current_values} ${last_values} + done + done + done + done + done +} + +function generate_html_body { +MR_TITLE='' +Test_Info_Title='' +Test_Info='' + +if [ "${qtools_branch}" == "" ]; +then + commit_id=$(echo ${ghprbActualCommit} |awk '{print substr($1,1,7)}') + + MR_TITLE="[ PR-${ghprbPullId} ]" + Test_Info_Title="Source Branch Target Branch Commit " + Test_Info="${MR_source_branch} ${MR_target_branch} ${commit_id}" +else + Test_Info_Title="Test Branch Commit ID " + Test_Info="${qtools_branch} ${qtools_commit} " +fi + +cat >> ${output_dir}/report.html << eof + + +
+

Neural Compressor Tuning Tests ${MR_TITLE} + [ Job-${BUILD_NUMBER} ]

+

Test Status: ${Jenkins_job_status}

+

Summary

+ + + + ${Test_Info_Title} + + + + ${Test_Info} + +
Repo
neural-compressor
+eof + + +echo "Generating benchmarks table" +cat >> ${output_dir}/report.html << eof +

Benchmark

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +eof +} + +function generate_html_footer { + + cat >> ${output_dir}/report.html << eof + + + + +
PlatformSystemFrameworkVersionModelVSTuning
Strategy
Tuning
Time(s)
Tuning
Count
INT8FP32Ratio
bsimgs/sbstop1bsimgs/sbstop1Throughput
INT8/FP32>=2
Accuracy
(INT8-FP32)/FP32>=-0.01
Note: All data tested on TensorFlow Dedicated Server.
+
+ + +eof +} + +function generate_html_head { + +cat > ${output_dir}/report.html << eof + + + + + + Daily Tests - TensorFlow - Jenkins + + + +eof + +} + +main diff --git a/.azure-pipelines/scripts/models/model_template.yml b/.azure-pipelines/scripts/models/model_template.yml new file mode 100644 index 00000000000..71cfc623a4a --- /dev/null +++ b/.azure-pipelines/scripts/models/model_template.yml @@ -0,0 +1,45 @@ + +parameters: +- name: modelName + type: string + default: 'resnet50v1.5' +- name: framework + type: string + default: 'tensorflow' + + +steps: +- script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: 'Clean workspace' +- checkout: self + clean: true + displayName: 'Checkout out Repo' +- script: | + if [[ ! $(docker images | grep -i neural-compressor:py38) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t neural-compressor:py38 . + fi + docker images | grep -i neural-compressor + if [[ $? -ne 0 ]]; then + echo "NO Such Image neural-compressor" + exit 1 + fi + displayName: "Build Devel Images" +- script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' +- script: | + echo "------ Run ${{ parameters.modelName }} Model Test ------" + docker run --disable-content-trust --privileged --name=${{ parameters.modelName }} --shm-size="2g" \ + -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 neural-compressor:py38 \ + /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --tune_acc=true --build_id=$(Build.BuildId)" + displayName: Tune&Benchmark ${{ parameters.modelName }} +- task: PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}/ + artifact: ${{ parameters.framework }}_${{ parameters.modelName }} + publishLocation: 'pipeline' diff --git a/.azure-pipelines/scripts/models/run_benchmark_common.sh b/.azure-pipelines/scripts/models/run_benchmark_common.sh new file mode 100644 index 00000000000..71653679878 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_benchmark_common.sh @@ -0,0 +1,96 @@ +#!/bin/bash +set -x + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --framework=*) + framework=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --input_model=*) + input_model=`echo $i | sed "s/${PATTERN}//"`;; + --benchmark_cmd=*) + benchmark_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --tune_acc=*) + tune_acc=`echo $i | sed "s/${PATTERN}//"`;; + --log_dir=*) + log_dir=`echo $i | sed "s/${PATTERN}//"`;; + --new_benchmark=*) + new_benchmark=`echo $i | sed "s/${PATTERN}//"`;; + --precision=*) + precision=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +echo "-------- run_benchmark_common --------" + +# run accuracy +# tune_acc==true means using accuracy results from tuning log +if [ "${tune_acc}" == "false" ]; then + echo "run tuning accuracy in precision ${precision}" + eval "${benchmark_cmd} --input_model=${input_model} --mode=accuracy" 2>&1 | tee ${log_dir}/${framework}-${model}-accuracy-${precision}.log +fi + + +function multiInstance() { + ncores_per_socket=${ncores_per_socket:=$( lscpu | grep 'Core(s) per socket' | cut -d: -f2 | xargs echo -n)} + echo "Executing multi instance benchmark" + ncores_per_instance=4 + echo "ncores_per_socket=${ncores_per_socket}, ncores_per_instance=${ncores_per_instance}" + + logFile="${log_dir}/${framework}-${model}-performance-${precision}" + benchmark_pids=() + + for((j=0;$j<${ncores_per_socket};j=$(($j + ${ncores_per_instance})))); + do + end_core_num=$((j + ncores_per_instance -1)) + if [ ${end_core_num} -ge ${ncores_per_socket} ]; then + end_core_num=$((ncores_per_socket-1)) + fi + numactl -m 0 -C "${j}-${end_core_num}" ${cmd} 2>&1 | tee ${logFile}-${ncores_per_socket}-${ncores_per_instance}-${j}.log & + benchmark_pids+=($!) + done + + status="SUCCESS" + for pid in "${benchmark_pids[@]}"; do + wait $pid + exit_code=$? + echo "Detected exit code: ${exit_code}" + if [ ${exit_code} == 0 ]; then + echo "Process ${pid} succeeded" + else + echo "Process ${pid} failed" + status="FAILURE" + fi + done + + echo "Benchmark process status: ${status}" + if [ ${status} == "FAILURE" ]; then + echo "Benchmark process returned non-zero exit code." + exit 1 + fi +} + + +# run performance +cmd="${benchmark_cmd} --input_model=${input_model}" + +if [ "${new_benchmark}" == "true" ]; then + echo "run performance in precision ${precision}" + eval ${cmd} 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log +else +# multiInstance + multiInstance 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log +fi + + + + + + diff --git a/.azure-pipelines/scripts/models/run_model_trigger_common.sh b/.azure-pipelines/scripts/models/run_model_trigger_common.sh new file mode 100644 index 00000000000..87133d70717 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_model_trigger_common.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --yaml=*) + yaml=`echo $i | sed "s/${PATTERN}//"`;; + --framework=*) + framework=`echo $i | sed "s/${PATTERN}//"`;; + --fwk_ver=*) + fwk_ver=`echo $i | sed "s/${PATTERN}//"`;; + --torch_vision_ver=*) + torch_vision_ver=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --model_src_dir=*) + model_src_dir=`echo $i | sed "s/${PATTERN}//"`;; + --dataset_location=*) + dataset_location=`echo $i | sed "s/${PATTERN}//"`;; + --input_model=*) + input_model=`echo $i | sed "s/${PATTERN}//"`;; + --batch_size=*) + batch_size=`echo $i | sed "s/${PATTERN}//"`;; + --strategy=*) + strategy=`echo $i | sed "s/${PATTERN}//"`;; + --new_benchmark=*) + new_benchmark=`echo $i | sed "s/${PATTERN}//"`;; + --tuning_cmd=*) + tuning_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --benchmark_cmd=*) + benchmark_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --tune_acc=*) + tune_acc=`echo $i | sed "s/${PATTERN}//"`;; + --build_id=*) + build_id=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +log_dir="/neural-compressor/.azure-pipelines/scripts/models" +WORK_SOURCE_DIR="/neural-compressor/examples/${framework}" +SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" +echo "processing ${framework}-${fwk_ver}-${model}" +echo "tuning_cmd is ${tuning_cmd}" +echo "benchmark_cmd is ${benchmark_cmd}" + +echo "======= creat log_dir =========" +if [ -d "${log_dir}/${model}" ]; then + echo "${log_dir}/${model} already exists, don't need to mkdir." +else + echo "no log dir ${log_dir}/${model}, create." + cd ${log_dir} + mkdir ${model} +fi + +echo "====== install requirements ======" +/bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh + +cd ${WORK_SOURCE_DIR}/${model_src_dir} +pip install ruamel_yaml +pip install psutil +pip install protobuf==3.20.1 +if [[ "${framework}" == "tensorflow" ]]; then + pip install intel-tensorflow==${fwk_ver} +elif [[ "${framework}" == "pytorch" ]]; then + pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html + pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html +elif [[ "${framework}" == "onnxrt" ]]; then + pip install onnx==${fwk_ver} + pip install onnxruntime==${fwk_ver} +elif [[ "${framework}" == "mxnet" ]]; then + if [[ "${fwk_ver}" == "1.7.0" ]]; then + pip install mxnet==${fwk_ver}.post2 + elif [[ "${fwk_ver}" == "1.6.0" ]]; then + pip install mxnet-mkl==${mxnet_version} + else + pip install mxnet==${fwk_ver} + fi +fi + +if [ -f "requirements.txt" ]; then + sed -i '/neural-compressor/d' requirements.txt + if [ "${framework}" == "onnxrt" ]; then + sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt + fi + if [ "${framework}" == "tensorflow" ]; then + sed -i '/tensorflow==/d;/tensorflow$/d' requirements.txt + sed -i '/^intel-tensorflow/d' requirements.txt + fi + if [ "${framework}" == "mxnet" ]; then + sed -i '/mxnet==/d;/mxnet$/d;/mxnet-mkl==/d;/mxnet-mkl$/d' requirements.txt + fi + if [ "${framework}" == "pytorch" ]; then + sed -i '/torch==/d;/torch$/d;/torchvision==/d;/torchvision$/d' requirements.txt + fi + n=0 + until [ "$n" -ge 5 ] + do + python -m pip install -r requirements.txt && break + n=$((n+1)) + sleep 5 + done + pip list +else + echo "Not found requirements.txt file." +fi + + +# ======== update yaml config ======== +echo -e "\nPrint origin yaml..." +cat ${yaml} +python ${SCRIPTS_PATH}/update_yaml_config.py --yaml=${yaml} --framework=${framework} \ +--dataset_location=${dataset_location} --batch_size=${batch_size} --strategy=${strategy} \ +--new_benchmark=${new_benchmark} --multi_instance='false' +echo -e "\nPrint updated yaml... " +cat ${yaml} + + +echo "======== run tuning ========" +/bin/bash ${SCRIPTS_PATH}/run_tuning_common.sh --framework=${framework} --model=${model} \ +--tuning_cmd="${tuning_cmd}" --log_dir="${log_dir}/${model}" --input_model=${input_model} --strategy=${strategy} \ +2>&1 | tee -a ${log_dir}/${model}/${framework}-${model}-tune.log + + +echo "====== run benchmark fp32 =======" +/bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh --framework=${framework} --model=${model} \ + --input_model=${input_model} --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} \ + --log_dir="${log_dir}/${model}" --new_benchmark=${new_benchmark} --precision="fp32" + + +echo "====== run benchmark int8 =======" +if [[ "${framework}" == "onnxrt" ]]; then + model_name="${log_dir}/${model}/${framework}-${model}-tune.onnx" +elif [[ "${framework}" == "mxnet" ]]; then + model_name="${log_dir}/${model}" +else + model_name="${log_dir}/${model}/${framework}-${model}-tune.pb" +fi +/bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh --framework=${framework} --model=${model} \ + --input_model="${model_name}" --benchmark_cmd="${benchmark_cmd}" \ + --tune_acc=${tune_acc} --log_dir="${log_dir}/${model}" --new_benchmark=${new_benchmark} --precision="int8" + + +echo "====== collect logs of model ${model} =======" +python -u ${SCRIPTS_PATH}/collect_log_model.py --framework=${framework} --fwk_ver=${fwk_ver} --model=${model} \ +--logs_dir="${log_dir}/${model}" --output_dir="${log_dir}/${model}" --build_id=${build_id} + + +echo "====== Finish model test =======" \ No newline at end of file diff --git a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh new file mode 100644 index 00000000000..e082f5189d9 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --tune_acc=*) + tune_acc=`echo $i | sed "s/${PATTERN}//"`;; + --build_id=*) + build_id=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="mxnet" +FRAMEWORK_VERSION="1.7.0" + + +# ======== set up config for mxnet models ======== +if [ "${model}" == "resnet50v1" ]; then + model_src_dir="image_recognition/cnn_models/quantization/ptq" + dataset_location="/tf_dataset/mxnet/val_256_q90.rec" + input_model="/tf_dataset/mxnet/resnet50_v1" + yaml="cnn.yaml" + strategy="mse" + batch_size=32 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=resnet50_v1 --dataset_location=${dataset_location} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=resnet50_v1 --dataset_location=${dataset_location} --batch_size=1 --iters=500 --mode=benchmark" +fi + + +/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ +--model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ +--input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ +--tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} --build_id=${build_id} diff --git a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh new file mode 100644 index 00000000000..d445067807f --- /dev/null +++ b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --tune_acc=*) + tune_acc=`echo $i | sed "s/${PATTERN}//"`;; + --build_id=*) + build_id=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="onnxrt" +FRAMEWORK_VERSION="1.11.0" + + +# ======== set up config for onnxrt models ======== +if [ "${model}" == "resnet50-v1-12" ]; then + model_src_dir="image_recognition/onnx_model_zoo/resnet50/quantization/ptq" + dataset_location="/tf_dataset2/datasets/imagenet/ImagenetRaw/ImagenetRaw_small_5000/ILSVRC2012_img_val" + input_model="/tf_dataset2/models/onnx/resnet50-v1-12/resnet50-v1-12.onnx" + yaml="resnet50_v1_5.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml}" +elif [ "${model}" == "bert_base_MRPC_static" ]; then + model_src_dir="language_translation/bert/quantization/ptq" + dataset_location="/tf_dataset/pytorch/glue_data/MRPC" + input_model="/tf_dataset2/models/onnx/bert_base_MRPC/bert.onnx" + yaml="bert_static.yaml" + strategy="basic" + batch_size=8 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml}" +elif [ "${model}" == "bert_base_MRPC_dynamic" ]; then + model_src_dir="language_translation/bert/quantization/ptq" + dataset_location="/tf_dataset/pytorch/glue_data/MRPC" + input_model="/tf_dataset2/models/onnx/bert_base_MRPC/bert.onnx" + yaml="bert_dynamic.yaml" + strategy="basic" + batch_size=8 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml}" +fi + + +/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ +--model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ +--input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ +--tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd} --mode=performance" --tune_acc=${tune_acc} --build_id=${build_id} diff --git a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh new file mode 100644 index 00000000000..795dd192ae7 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --tune_acc=*) + tune_acc=`echo $i | sed "s/${PATTERN}//"`;; + --build_id=*) + build_id=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="pytorch" +FRAMEWORK_VERSION="1.12.0+cpu" +TORCH_VISION_VERSION="0.13.0+cpu" + + +# ======== set up config for pytorch models ======== +if [ "${model}" == "resnet18" ]; then + model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/eager" + dataset_location="/tf_dataset2/datasets/mini-imageraw" + input_model="" + yaml="conf.yaml" + strategy="bayesian" + batch_size=100 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=200" +elif [ "${model}" == "resnet18_fx" ]; then + model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/fx/" + dataset_location="/tf_dataset2/datasets/mini-imageraw" + input_model="" + yaml="conf.yaml" + strategy="basic" + batch_size=100 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=200" +fi + + +/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ +--torch_vision_ver=${TORCH_VISION_VERSION} --model=${model} --model_src_dir=${model_src_dir} \ +--dataset_location=${dataset_location} --input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} \ +--new_benchmark=${new_benchmark} --tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" \ +--tune_acc=${tune_acc} --build_id=${build_id} diff --git a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh new file mode 100644 index 00000000000..a3cccb37e28 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@" +do + case $i in + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --tune_acc=*) + tune_acc=`echo $i | sed "s/${PATTERN}//"`;; + --build_id=*) + build_id=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +FRAMEWORK="tensorflow" +FRAMEWORK_VERSION="2.9.1" + +# ======== set up config for tensorflow models ======== +if [ "${model}" == "resnet50v1.5" ]; then + model_src_dir="image_recognition/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/dataset/TF_mini_imagenet" + input_model="/tf_dataset/pre-trained-models/resnet50v1_5/fp32/resnet50_v1.pb" + yaml="resnet50_v1_5.yaml" + strategy="basic" + batch_size=100 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "ssd_resnet50_v1" ];then + model_src_dir="object_detection/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/tensorflow/mini-coco-100.record" + input_model="/tf_dataset/pre-train-model-oob/object_detection/ssd_resnet50_v1/frozen_inference_graph.pb" + yaml="ssd_resnet50_v1.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "ssd_mobilenet_v1_ckpt" ];then + model_src_dir="object_detection/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/tensorflow/mini-coco-100.record" + input_model="/tf_dataset/pre-train-model-oob/object_detection/ssd_mobilenet_v1" + yaml="ssd_mobilenet_v1.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "inception_v1" ]; then + model_src_dir="image_recognition/tensorflow_models/quantization/ptq" + dataset_location="/tf_dataset/dataset/TF_mini_imagenet" + input_model="/tf_dataset/pre-train-model-slim/pbfile/frozen_pb/frozen_inception_v1.pb" + yaml="inception_v1.yaml" + strategy="basic" + batch_size=100 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +elif [ "${model}" == "darknet19" ]; then + model_src_dir="oob_models/quantization/ptq" + dataset_location="" + input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/PublicInHouse/classification/darknet19/darknet19.pb" + yaml="config.yaml" + strategy="basic" + batch_size=100 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=200" +elif [ "${model}" == "densenet-121" ]; then + model_src_dir="oob_models/quantization/ptq" + dataset_location="" + input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/classification/densenet/121/tf/densenet-121.pb" + yaml="config.yaml" + strategy="basic" + batch_size=100 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=200" +elif [ "${model}" == "resnet-101" ]; then + model_src_dir="oob_models/quantization/ptq" + dataset_location="" + input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/classification/resnet/v1/101/tf/resnet-101.pb" + yaml="config.yaml" + strategy="basic" + batch_size=1 + new_benchmark=false + tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=200" +elif [ "${model}" == "resnet50_fashion" ]; then + model_src_dir="image_recognition/keras_models/resnet50_fashion/quantization/ptq" + dataset_location="/tf_dataset2/datasets/mnist/FashionMNIST_small" + input_model="/tf_dataset2/models/tensorflow/resnet50_fashion" + yaml="resnet50_fashion.yaml" + strategy="basic" + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" +fi + + +/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ +--model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ +--input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ +--tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} --build_id=${build_id} + diff --git a/.azure-pipelines/scripts/models/run_tuning_common.sh b/.azure-pipelines/scripts/models/run_tuning_common.sh new file mode 100644 index 00000000000..a65d639d1d7 --- /dev/null +++ b/.azure-pipelines/scripts/models/run_tuning_common.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -x + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +starttime=`date +'%Y-%m-%d %H:%M:%S'` + +for i in "$@" +do + case $i in + --framework=*) + framework=`echo $i | sed "s/${PATTERN}//"`;; + --model=*) + model=`echo $i | sed "s/${PATTERN}//"`;; + --input_model=*) + input_model=`echo $i | sed "s/${PATTERN}//"`;; + --tuning_cmd=*) + tuning_cmd=`echo $i | sed "s/${PATTERN}//"`;; + --log_dir=*) + log_dir=`echo $i | sed "s/${PATTERN}//"`;; + --strategy=*) + strategy=`echo $i | sed "s/${PATTERN}//"`;; + *) + echo "Parameter $i not recognized."; exit 1;; + esac +done + +# run tuning +if [ "${framework}" == "onnxrt" ]; then + output_model=${log_dir}/${framework}-${model}-tune.onnx +elif [ "${framework}" == "mxnet" ]; then + output_model=${log_dir}/resnet50_v1 +else + output_model=${log_dir}/${framework}-${model}-tune.pb +fi + +echo -e "-------- run_tuning_common --------" +echo ${tuning_cmd} +eval "/usr/bin/time -v ${tuning_cmd} --output_model=${output_model}" + +echo "====== finish tuning. echo information. ======" +endtime=`date +'%Y-%m-%d %H:%M:%S'` +start_seconds=$(date --date="$starttime" +%s); +end_seconds=$(date --date="$endtime" +%s); +echo "Tuning time spend: "$((end_seconds-start_seconds))"s " + +echo "Tuning strategy: ${strategy}" + +echo "Total resident size (kbytes): $(cat /proc/meminfo |grep 'MemTotal' |sed 's/[^0-9]//g')" + diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py new file mode 100644 index 00000000000..0e3d487255e --- /dev/null +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -0,0 +1,332 @@ +import argparse +import re +import os +import psutil +from typing import Optional, Union + +import platform + +system = platform.system() +try: + import ruamel.yaml as yaml +except: + import ruamel_yaml as yaml + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--yaml", type=str, required=True, help="Path to yaml config.") + parser.add_argument("--framework", type=str, required=True, help="Framework of model.") + parser.add_argument("--dataset_location", type=str, required=True, help="Location of dataset used for model.") + parser.add_argument("--strategy", type=str, required=False, help="Strategy to update.") + parser.add_argument("--batch_size", type=int, required=False, help="Batch size.") + parser.add_argument("--new_benchmark", type=str, required=False, help="Whether to modify benchmark config.") + parser.add_argument("--multi_instance", type=str, required=False, help="Whether to eval in multi-instance.") + return parser.parse_args() + + +def update_yaml_dataset(yaml, framework, dataset_location): + if not os.path.isfile(yaml): + raise Exception(f"Not found yaml config at '{yaml}' location.") + + print("Reading config") + with open(yaml, "r") as config: + lines = config.readlines() + + # Update dataset + if framework != "pytorch": + val_txt_location = os.path.dirname(dataset_location) + f"{os.path.sep}" + "val.txt" + + patterns = { + "root_path": { + "pattern": r'root:.*/path/to/(calibration|evaluation)/dataset/?', + "replacement": f"root: {dataset_location}", + }, + "data_path": { + "pattern": r'data_path:.*/path/to/(calibration|evaluation)/dataset/?', + "replacement": f"data_path: {dataset_location}", + }, + "image_list": { + "pattern": r'image_list:.*/path/to/(calibration|evaluation)/label/?', + "replacement": f"image_list: {val_txt_location}", + }, + "data_dir": { + "pattern": r'data_dir:.*/path/to/dataset/?', + "replacement": f"data_dir: {dataset_location}", + }, + } + print("======= update_yaml_dataset =======") + with open(yaml, "w") as config: + for line in lines: + for key, key_patterns in patterns.items(): + if re.search(key_patterns["pattern"], line): + print(f"Replacing {key} key.") + line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) + # print(line) + config.write(line) + + if framework == "pytorch": + val_dataset = dataset_location + f"\{os.path.sep}" + "val" + train_dataset = dataset_location + f"\{os.path.sep}" + "train" + patterns = { + "calibration_dataset": { + "pattern": r'root:.*/path/to/calibration/dataset/?', + "replacement": f"root: {train_dataset}", + }, + "evaluation_dataset": { + "pattern": r'root:.*/path/to/evaluation/dataset/?', + "replacement": f"root: {val_dataset}", + }, + } + + if topology == "distilbert_base_MRPC": + patterns.update({ + "bert_name": { + "pattern": r'name:/s+bert', + "replacement": "name: distilbert", + } + }) + + print("======= update_yaml_dataset =======") + with open(yaml, "w") as config: + for line in lines: + for key, key_patterns in patterns.items(): + if re.search(key_patterns["pattern"], line): + print(f"Replacing {key} key.") + line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) + # print(line) + config.write(line) + + +def update_yaml_config_tuning(yaml_file, strategy = None, mode = None, batch_size = None, iteration = None, + max_trials = None, algorithm = None, timeout = None, strategy_token = None, + sampling_size = None, dtype = None, tf_new_api = None): + with open(yaml_file) as f: + yaml_config = yaml.round_trip_load(f, preserve_quotes=True) + + if algorithm: + try: + model_wise = yaml_config.get("quantization", {}).get("model_wise", {}) + prev_activation = model_wise.get("activation", {}) + if not prev_activation: + model_wise.update({"activation": {}}) + prev_activation = model_wise.get("activation", {}) + prev_activation.update({"algorithm": algorithm}) + except Exception as e: + print(f"[ WARNING ] {e}") + + if timeout: + try: + exit_policy = yaml_config.get("tuning", {}).get("exit_policy", {}) + prev_timeout = exit_policy.get("timeout", None) + exit_policy.update({"timeout": timeout}) + print(f"Changed {prev_timeout} to {timeout}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if strategy and strategy != "basic": # Workaround for PyTorch huggingface models (`sed` in run_tuning.sh) + try: + tuning_config = yaml_config.get("tuning", {}) + prev_strategy = tuning_config.get("strategy", {}) + if not prev_strategy: + tuning_config.update({"strategy": {}}) + prev_strategy = tuning_config.get("strategy", {}) + strategy_name = prev_strategy.get("name", None) + prev_strategy.update({"name": strategy}) + if strategy == "sigopt": + prev_strategy.update({ + "sigopt_api_token": strategy_token, + "sigopt_project_id": "lpot", + "sigopt_experiment_name": "lpot-tune", + }) + if strategy == "hawq": + prev_strategy.update({"loss": "CrossEntropyLoss"}) + print(f"Changed {strategy_name} to {strategy}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if max_trials and max_trials > 0: + try: + tuning_config = yaml_config.get("tuning", {}) + prev_exit_policy = tuning_config.get("exit_policy", {}) + if not prev_exit_policy: + tuning_config.update({"exit_policy": { + "max_trials": max_trials + }}) + else: + prev_max_trials = prev_exit_policy.get("max_trials", None) + prev_exit_policy.update({"max_trials": max_trials}) + print(f"Changed {prev_max_trials} to {max_trials}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if mode == 'accuracy': + try: + # delete performance part in yaml if exist + performance = yaml_config.get("evaluation", {}).get("performance", {}) + if performance: + yaml_config.get("evaluation", {}).pop("performance", {}) + # accuracy batch_size replace + if batch_size: + try: + dataloader = yaml_config.get("evaluation", {}).get("accuracy", {}).get("dataloader", {}) + prev_batch_size = dataloader.get("batch_size", None) + dataloader.update({"batch_size": batch_size}) + print(f"Changed accuracy batch size from {prev_batch_size} to {batch_size}") + except Exception as e: + print(f"[ WARNING ] {e}") + except Exception as e: + print(f"[ WARNING ] {e}") + elif mode: + try: + # delete accuracy part in yaml if exist + accuracy = yaml_config.get("evaluation", {}).get("accuracy", {}) + if accuracy: + yaml_config.get("evaluation", {}).pop("accuracy", {}) + # performance iteration replace + if iteration: + try: + performance = yaml_config.get("evaluation", {}).get("performance", {}) + prev_iteration = performance.get("iteration", None) + performance.update({"iteration": iteration}) + print(f"Changed performance batch size from {prev_iteration} to {iteration}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if batch_size and mode == 'latency': + try: + dataloader = yaml_config.get("evaluation", {}).get("performance", {}).get("dataloader", {}) + prev_batch_size = dataloader.get("batch_size", None) + dataloader.update({"batch_size": batch_size}) + print(f"Changed accuracy batch size from {prev_batch_size} to {batch_size}") + except Exception as e: + print(f"[ WARNING ] {e}") + + except Exception as e: + print(f"[ WARNING ] {e}") + + if sampling_size: + try: + calibration = yaml_config.get("quantization", {}).get("calibration", {}) + prev_sampling_size = calibration.get("sampling_size", None) + calibration.update({"sampling_size": sampling_size}) + print(f"Changed calibration sampling size from {prev_sampling_size} to {sampling_size}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if dtype: + try: + quantization = yaml_config.get("quantization", {}) + prev_dtype = quantization.get("dtype", None) + quantization.update({"dtype": dtype}) + print(f"Changed dtype from {prev_dtype} to {dtype}") + except Exception as e: + print(f"[ WARNING ] {e}") + + if tf_new_api == "true": + try: + model = yaml_config.get("model", {}) + prev_framework = model.get("framework", None) + model.update({"framework": "inteltensorflow"}) + print(f"Changed framework from {prev_framework} to inteltensorflow") + except Exception as e: + print(f"[ WARNING ] {e}") + + print(f"====== update_yaml_config_tuning ========") + + yaml_content = yaml.round_trip_dump(yaml_config) + + # print(yaml_content) + + with open(yaml_file, 'w') as output_file: + output_file.write(yaml_content) + + +def update_yaml_config_benchmark_acc(yaml_path: str, batch_size = None): + with open(yaml_path) as f: + yaml_config = yaml.round_trip_load(f, preserve_quotes=True) + try: + accuracy = yaml_config.get("evaluation", {}).get("accuracy", {}) + if not accuracy: + raise AttributeError + dataloader = accuracy.get('dataloader', {}) + if dataloader: + dataloader.update({'batch_size': batch_size}) + configs = accuracy.get('configs', {}) + if configs: + del accuracy['configs'] + except Exception as e: + print(f"[ WARNING ] {e}") + + print(f"====== update_yaml_config_benchmark_acc ========") + + yaml_content = yaml.round_trip_dump(yaml_config) + + # print(yaml_content) + + with open(yaml_path, 'w') as output_file: + output_file.write(yaml_content) + + +def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_instance = None): + # Get cpu information for multi-instance + total_cores = psutil.cpu_count(logical=False) + # total_sockets = get_number_of_sockets() + total_sockets = 1 + ncores_per_socket = total_cores / total_sockets + + num_sockets = 1 # Use only one socket + num_benchmark_cores = ncores_per_socket * num_sockets + + ncores_per_instance = ncores_per_socket + iters = 100 + + if multi_instance=='true': + ncores_per_instance = 4 + iters = 500 + + with open(yaml_path) as f: + yaml_config = yaml.round_trip_load(f, preserve_quotes=True) + try: + performance = yaml_config.get("evaluation", {}).get("performance", {}) + if not performance: + raise AttributeError + dataloader = performance.get('dataloader', {}) + if dataloader: + dataloader.update({'batch_size': batch_size}) + performance.update({'iteration': iters}) + configs = performance.get('configs', {}) + if not configs: + raise AttributeError + else: + configs.update({ + 'cores_per_instance': int(ncores_per_instance), + 'num_of_instance': int(num_benchmark_cores // ncores_per_instance) + }) + for attr in ['intra_num_of_threads', 'inter_num_of_threads', 'kmp_blocktime']: + if configs.get(attr): + del configs[attr] + print(configs) + except Exception as e: + print(f"[ WARNING ] {e}") + + print(f"====== update_yaml_config_benchmark_perf ========") + + yaml_content = yaml.round_trip_dump(yaml_config) + + # print(yaml_content) + + with open(yaml_path, 'w') as output_file: + output_file.write(yaml_content) + + +if __name__ == "__main__": + args = parse_args() + update_yaml_dataset(args.yaml, args.framework, args.dataset_location) + update_yaml_config_tuning(args.yaml, strategy=args.strategy) + print('===== multi_instance={} ===='.format(args.multi_instance)) + if args.new_benchmark=='true': + update_yaml_config_benchmark_acc(args.yaml, batch_size=args.batch_size) + update_yaml_config_benchmark_perf(args.yaml, batch_size=args.batch_size, multi_instance=args.multi_instance) + + From ac95b65b44600fcfbd1d8e2924258d25f464ff7d Mon Sep 17 00:00:00 2001 From: VincyZhang Date: Tue, 13 Sep 2022 15:51:59 +0800 Subject: [PATCH 276/296] Ut coverage combine (#23) --- .azure-pipelines/scripts/ut/collect_log.sh | 61 ++++ .../scripts/ut/compare_coverage.sh | 238 ++++++++++++ .azure-pipelines/scripts/ut/coverage.file | 25 ++ .azure-pipelines/scripts/ut/env_setup.sh | 3 + .../scripts/ut/run_basic_adaptor.sh | 8 +- .../scripts/ut/run_basic_adaptor_tfnewapi.sh | 8 +- .azure-pipelines/scripts/ut/run_basic_ipex.sh | 8 +- .../scripts/ut/run_basic_others.sh | 10 +- .azure-pipelines/ut-basic.yml | 343 ++++++++++++++++-- 9 files changed, 657 insertions(+), 47 deletions(-) create mode 100644 .azure-pipelines/scripts/ut/collect_log.sh create mode 100644 .azure-pipelines/scripts/ut/compare_coverage.sh create mode 100644 .azure-pipelines/scripts/ut/coverage.file diff --git a/.azure-pipelines/scripts/ut/collect_log.sh b/.azure-pipelines/scripts/ut/collect_log.sh new file mode 100644 index 00000000000..58900c280d4 --- /dev/null +++ b/.azure-pipelines/scripts/ut/collect_log.sh @@ -0,0 +1,61 @@ +pip install coverage +export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/coverage.file +coverage_log="/neural-compressor/log_dir/coverage_log" +coverage_log_base="/neural-compressor/log_dir/coverage_log_base" +coverage_compare="/neural-compressor/log_dir/coverate_compare.html" +cd /neural-compressor/log_dir +echo "collect coverage for PR branch" +mkdir -p coverage_PR +cp ut-coverage-adaptor/.coverage.adaptor ./coverage_PR/ +cp ut-coverage-tfnewapi/.coverage.tfnewapi ./coverage_PR/ +cp ut-coverage-others/.coverage.others ./coverage_PR/ +cp ut-coverage-ipex/.coverage.ipex ./coverage_PR/ +cd coverage_PR +coverage combine --keep --rcfile=${COVERAGE_RCFILE} +cp .coverage /neural-compressor/.coverage +cd /neural-compressor +coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log} +coverage html -d log_dir/coverage_PR/htmlcov --rcfile=${COVERAGE_RCFILE} +coverage xml -o log_dir/coverage_PR/coverage.xml --rcfile=${COVERAGE_RCFILE} +ls -l log_dir/coverage_PR/htmlcov +echo "collect coverage for baseline" +coverage erase +cd /neural-compressor/log_dir +mkdir -p coverage_base +cp ut-coverage-adaptor-base/.coverage.adaptor ./coverage_base/ +cp ut-coverage-tfnewapi-base/.coverage.tfnewapi ./coverage_base/ +cp ut-coverage-others-base/.coverage.others ./coverage_base/ +cp ut-coverage-ipex-base/.coverage.ipex ./coverage_base/ +cd coverage_base +coverage combine --keep --rcfile=${COVERAGE_RCFILE} +cp .coverage /neural-compressor/.coverage +cd /neural-compressor +coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log_base} +coverage html -d log_dir/coverage_base/htmlcov --rcfile=${COVERAGE_RCFILE} +coverage xml -o log_dir/coverage_base/coverage.xml --rcfile=${COVERAGE_RCFILE} +ls -l log_dir/coverage_base/htmlcov +echo "compare coverage" +coverage_PR_total=$(cat ${coverage_log} | grep TOTAL | awk '{print $NF}' | sed "s|%||g") +coverage_base_total=$(cat ${coverage_log_base} | grep TOTAL | awk '{print $NF}' | sed "s|%||g") +echo "clear upload path" +rm -fr log_dir/coverage_PR/.coverage* +rm -fr log_dir/coverage_base/.coverage* +rm -fr log_dir/ut-coverage-* +if [[ ${coverage_PR_total} -lt ${coverage_base_total} ]]; then + decreate=$(($coverage_PR_total - $coverage_base_total)) + rate=$(awk 'BEGIN{printf "%.2f%\n",'$decreate/100'}') + echo "Unit Test failed with covereage decrese ${rate}%" + echo "compare coverage to give detail info" + bash -x /neural-compressor/.azure-pipelines/scripts/ut/compare_coverage.sh ${coverage_compare} ${coverage_log} ${coverage_log_base} "FAILED" + exit 1 +else + echo "Unit Test success with coverage ${coverage_PR_total}%" + echo "compare coverage to give detail info" + bash -x /neural-compressor/.azure-pipelines/scripts/ut/compare_coverage.sh ${coverage_compare} ${coverage_log} ${coverage_log_base} "SUCCESS" + #sed "1i\Unit Test success with coverage ${coverage_PR_total}\n" ${coverage_log} +fi + + +#rm -r ${coverage_log} +#rm -r ${coverage_log_base} + diff --git a/.azure-pipelines/scripts/ut/compare_coverage.sh b/.azure-pipelines/scripts/ut/compare_coverage.sh new file mode 100644 index 00000000000..164593af2fe --- /dev/null +++ b/.azure-pipelines/scripts/ut/compare_coverage.sh @@ -0,0 +1,238 @@ +output_file=$1 +coverage_pr_log=$2 +coverage_base_log=$3 +coverage_status=$4 +module_name="neural_compressor" +[[ ! -f $coverage_pr_log ]] && exit 1 +[[ ! -f $coverage_base_log ]] && exit 1 +file_name="./coverage_compare" +sed -i "s|\/usr.*${module_name}\/||g" $coverage_pr_log +sed -i "s|\/usr.*${module_name}\/||g" $coverage_base_log +diff $coverage_pr_log $coverage_base_log > diff_file +[[ $? == 0 ]] && exit 0 +grep -Po "[<,>,\d].*" diff_file | awk '{print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7}' | sed "/Name/d" | sed "/TOTAL/d" |sed "/---/d" > $file_name +[[ ! -s $file_name ]] && exit 0 +[[ -f $output_file ]] && rm -f $output_file +touch $output_file + +function generate_html_head { + +cat > ${output_file} << eof + + + + + + Daily Tests - TensorFlow - Jenkins + + + +eof +} + +function main { + generate_html_head + # generate table head + PR_stmt=$(grep "TOTAL" $coverage_pr_log | awk '{print $2}') + PR_miss=$(grep "TOTAL" $coverage_pr_log | awk '{print $3}') + PR_cover=$(grep "TOTAL" $coverage_pr_log | awk '{print $NF}') + BASE_stmt=$(grep "TOTAL" $coverage_base_log | awk '{print $2}') + BASE_miss=$(grep "TOTAL" $coverage_base_log | awk '{print $3}') + BASE_cover=$(grep "TOTAL" $coverage_base_log | awk '{print $NF}') + echo """ + +
+

Coverage Summary : ${coverage_status}

+ + + + + + + + + + + + + + + + + + + +
CommitStatementsMissCoverage
PR ${PR_stmt} ${PR_miss} ${PR_cover}
BASE ${BASE_stmt} ${BASE_miss} ${BASE_cover}
+
+ """ >> ${output_file} + if [[ ${coverage_status} = "SUCCESS" ]]; then + echo """""" >> ${output_file} + echo "coverage PASS, no need to compare difference" + exit 0 + fi + echo """ +
+

Coverage Detail

+ + + + + + + + + """ >> ${output_file} + # generate compare detail + cat $file_name | while read line + do + if [[ $(echo $line | grep "[0-9]a[0-9]") ]] && [[ $(grep -A 1 "$line" $file_name | grep ">") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep ">") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po ">.*[a-z,A-Z].*.py" | sed "s|>||g") + for diff_file in ${diff_file_name} + do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + file=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $1}') + miss=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $3}') + cover=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $6}') + branch=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $4}') + echo """ + + + + + """ >> ${output_file} + done + elif [[ $(echo $line | grep "[0-9]c[0-9]") ]] && [[ $(cat $file_name | grep -A 1 "$line" | grep "<") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g") + #diff_file_name=$(echo ${diff_lines} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|,||g) + for diff_file in ${diff_file_name} + do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + file1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $1}') + miss1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $3}') + cover1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $6}') + branch1=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $4}') + file2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $1}') + miss2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $3}') + cover2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $6}') + branch2=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po ">.*" | sed 's/>[ \t]*//g' | awk '{print $4}') + # if branch coverage not change, not consider as regression + [[ "${branch1}" == "${branch2}" ]] && continue + echo """ + + + + + """ >> ${output_file} + done + elif [[ $(echo $line | grep "[0-9]d[0-9]") ]] && [[ $(cat $file_name | grep -A 1 "$line" | grep "<") ]]; then + diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<") + diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g") + for diff_file in ${diff_file_name} + do + diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g') + file=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $1}') + miss=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $3}') + cover=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $6}') + branch=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "<.*" | sed 's/<[ \t]*//g' | awk '{print $4}') + echo """ + + + + + """ >> ${output_file} + done + fi + done + # generage table end + echo """
CommitFileNameMissBranchCover
PR | BASE${file}NA | ${miss}NA | ${branch}NA | ${cover}
PR | BASE${file1}${miss1} | ${miss2}${branch1} | ${branch2}${cover1} | ${cover2}
PR | BASE${file} | NA${miss} | NA${branch} | NA${cover} | NA
""" >> ${output_file} + +} + +main + diff --git a/.azure-pipelines/scripts/ut/coverage.file b/.azure-pipelines/scripts/ut/coverage.file new file mode 100644 index 00000000000..4e649650502 --- /dev/null +++ b/.azure-pipelines/scripts/ut/coverage.file @@ -0,0 +1,25 @@ +[run] +branch = True +relative_files=True +[paths] +source = + /usr/local/lib/python3.8/dist-packages/neural_compressor + /neural-compressor/neural_compressor +[report] +omit = + */**/fake*yaml + */**/fake.py + */neural_compressor/ux/* + */neural_compressor/model/nets_factory.py + */neural_compressor/experimental/benchmark.py + */neural_compressor/contrib/strategy/tpe.py + */nlp_toolkit/backends/* +exclude_lines = + pragma: no cover + raise NotImplementedError + raise TypeError + if self.device == "gpu": + if device == "gpu": + except ImportError: + onnx_version < ONNX18_VERSION + onnx_version >= ONNX18_VERSION diff --git a/.azure-pipelines/scripts/ut/env_setup.sh b/.azure-pipelines/scripts/ut/env_setup.sh index 9c101916a1b..096198ad3d5 100644 --- a/.azure-pipelines/scripts/ut/env_setup.sh +++ b/.azure-pipelines/scripts/ut/env_setup.sh @@ -70,3 +70,6 @@ if [ -f "requirements.txt" ]; then else echo "Not found requirements.txt file." fi + +pip install coverage +pip install pytest diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh index 2a7fdcb283c..6600029b1ad 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor.sh @@ -11,9 +11,9 @@ export mxnet_version='1.7.0' echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh - +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') cd /neural-compressor/test || exit 1 -find ./adaptor -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +find ./adaptor -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh LOG_DIR=/neural-compressor/log_dir mkdir -p ${LOG_DIR} @@ -23,7 +23,9 @@ echo "cat run.sh..." cat run.sh | tee ${ut_log_name} echo "-------------" bash run.sh 2>&1 | tee -a ${ut_log_name} - +cp .coverage ${LOG_DIR}/.coverage.adaptor +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then exit 1 fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh index 2c9fbf37649..2b687e633d3 100644 --- a/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh +++ b/.azure-pipelines/scripts/ut/run_basic_adaptor_tfnewapi.sh @@ -9,9 +9,9 @@ export tensorflow_version='spr-base' echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh - +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') cd /neural-compressor/test || exit 1 -find ./tfnewapi -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +find ./tfnewapi -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh LOG_DIR=/neural-compressor/log_dir mkdir -p ${LOG_DIR} @@ -21,7 +21,9 @@ echo "cat run.sh..." cat run.sh | tee ${ut_log_name} echo "-------------" bash run.sh 2>&1 | tee -a ${ut_log_name} - +cp .coverage ${LOG_DIR}/.coverage.tfnewapi +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then exit 1 fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_ipex.sh b/.azure-pipelines/scripts/ut/run_basic_ipex.sh index c005e957238..c5742a75e8c 100644 --- a/.azure-pipelines/scripts/ut/run_basic_ipex.sh +++ b/.azure-pipelines/scripts/ut/run_basic_ipex.sh @@ -10,9 +10,9 @@ export ipex_version='1.12.0+cpu' echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh - +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') cd /neural-compressor/test || exit 1 -find ./ipex -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +find ./ipex -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh LOG_DIR=/neural-compressor/log_dir mkdir -p ${LOG_DIR} @@ -22,7 +22,9 @@ echo "cat run.sh..." cat run.sh | tee ${ut_log_name} echo "-------------" bash run.sh 2>&1 | tee -a ${ut_log_name} - +cp .coverage ${LOG_DIR}/.coverage.ipex +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then exit 1 fi \ No newline at end of file diff --git a/.azure-pipelines/scripts/ut/run_basic_others.sh b/.azure-pipelines/scripts/ut/run_basic_others.sh index 53b61269c7a..0e5e91496e9 100644 --- a/.azure-pipelines/scripts/ut/run_basic_others.sh +++ b/.azure-pipelines/scripts/ut/run_basic_others.sh @@ -13,17 +13,21 @@ export mxnet_version='1.7.0' echo "set up UT env..." bash /neural-compressor/.azure-pipelines/scripts/ut/env_setup.sh +lpot_path=$(python -c 'import neural_compressor; import os; print(os.path.dirname(neural_compressor.__file__))') echo "copy pre-train model..." mkdir -p /tmp/.neural_compressor/inc_ut || true cp -r /tf_dataset/ut-localfile/resnet_v2 /tmp/.neural_compressor/inc_ut || true cd /neural-compressor/test || exit 1 -find . -name "test*.py" | sed 's,\.\/,python ,g' | sed 's/$/ --verbose/' > run.sh +find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${lpot_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh sed -i '/ adaptor\//d' run.sh sed -i '/ tfnewapi\//d' run.sh sed -i '/ ux\//d' run.sh sed -i '/ neural_coder\//d' run.sh sed -i '/ ipex\//d' run.sh +sed -i '/ pruning\//d' run.sh +sed -i '/ distillation\//d' run.sh +sed -i '/ nas\//d' run.sh LOG_DIR=/neural-compressor/log_dir mkdir -p ${LOG_DIR} @@ -33,7 +37,9 @@ echo "cat run.sh..." cat run.sh | tee ${ut_log_name} echo "-------------" bash run.sh 2>&1 | tee -a ${ut_log_name} - +cp .coverage ${LOG_DIR}/.coverage.others +echo "list all in ${LOG_DIR}" +ls -a ${LOG_DIR} if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then exit 1 fi \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index cf19975eb78..1fc05f2564d 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -15,9 +15,12 @@ pool: ICX-16C variables: IMAGE_NAME: 'neural-compressor' IMAGE_TAG: 'py38' + UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir + DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir + ARTIFACT_NAME: 'UT_coverage_report' stages: -- stage: +- stage: Adaptor displayName: Unit Test FWKs adaptor dependsOn: [] jobs: @@ -54,8 +57,13 @@ stages: && bash install_nc.sh \ && bash ut/run_basic_adaptor.sh" displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-adaptor' + publishLocation: 'pipeline' -- stage: +- stage: TFNewAPI displayName: Unit Test tf newAPI dependsOn: [] jobs: @@ -92,47 +100,97 @@ stages: && bash install_nc.sh \ && bash ut/run_basic_adaptor_tfnewapi.sh" displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-tfnewapi' + publishLocation: 'pipeline' -- stage: +- stage: IPEX displayName: Unit Test IPEX dependsOn: [] jobs: - - job: - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' + - checkout: self + displayName: "Checkout out Repo" + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_ipex.sh" + displayName: 'Install Neural Compressor and run UT' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-ipex' + publishLocation: 'pipeline' - - checkout: self - displayName: "Checkout out Repo" +- stage: Others + displayName: Unit Test other basic case + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + displayName: 'Clean workspace' - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" + - checkout: self + displayName: "Checkout out Repo" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_ipex.sh" - displayName: 'Install Neural Compressor and run UT' + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' -- stage: - displayName: Unit Test other basic case + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_others.sh" + displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-others' + publishLocation: 'pipeline' + +- stage: Adaptor_base + displayName: Unit Test FWKs adaptor baseline dependsOn: [] jobs: - job: @@ -140,10 +198,168 @@ stages: - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo mkdir ${BUILD_SOURCESDIRECTORY} + sudo chmod 777 ${BUILD_SOURCESDIRECTORY} displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" + - checkout: none + + - script: | + git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} + git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git checkout master + #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* + displayName: "Checkout out master" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor.sh" + displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-adaptor-base' + publishLocation: 'pipeline' +- stage: TFNewAPI_base + displayName: Unit Test tf newAPI baseline + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo mkdir ${BUILD_SOURCESDIRECTORY} + sudo chmod 777 ${BUILD_SOURCESDIRECTORY} + displayName: 'Clean workspace' + + - checkout: none + - script: | + git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} + git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git checkout master + #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* + displayName: "Checkout out master" + + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_adaptor_tfnewapi.sh" + displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-tfnewapi-base' + publishLocation: 'pipeline' + +- stage: IPEX_base + displayName: Unit Test IPEX baseline + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo mkdir ${BUILD_SOURCESDIRECTORY} + sudo chmod 777 ${BUILD_SOURCESDIRECTORY} + displayName: 'Clean workspace' + - checkout: none + - script: | + git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} + git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git checkout master + #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* + displayName: "Checkout out master" + - script: | + if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build develop docker image" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean docker' + - script: | + docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ + bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/run_basic_ipex.sh" + displayName: 'Install Neural Compressor and run UT' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-ipex-base' + publishLocation: 'pipeline' + +- stage: Others_base + displayName: Unit Test other cases baseline + dependsOn: [] + jobs: + - job: + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo mkdir ${BUILD_SOURCESDIRECTORY} + sudo chmod 777 ${BUILD_SOURCESDIRECTORY} + displayName: 'Clean workspace' + + - checkout: none + - script: | + git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} + git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git checkout master + #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* + displayName: "Checkout out master" - script: | if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then @@ -168,3 +384,58 @@ stages: && bash install_nc.sh \ && bash ut/run_basic_others.sh" displayName: 'Install Neural Compressor and run UT' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: 'ut-coverage-others-base' + publishLocation: 'pipeline' + +- stage: Coverage + displayName: "Coverage Combine" + dependsOn: [Adaptor, TFNewAPI, Others, IPEX, Adaptor_base, TFNewAPI_base, Others_base, IPEX_base] + jobs: + - job: CollectDatafiles + displayName: collect coverage files + steps: + - script: | + echo ${BUILD_SOURCESDIRECTORY} + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: 'Clean workspace' + - checkout: self + clean: true + displayName: 'Checkout out Repo' + - script: | + if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . + fi + docker images | grep -i ${IMAGE_NAME} + if [[ $? -ne 0 ]]; then + echo "NO Such Image ${IMAGE_NAME}" + exit 1 + fi + displayName: "Build Devel Images" + - script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + displayName: 'Clean Docker' + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + path: $(DOWNLOAD_PATH) + - script: | + echo "------ Collecting logs ------" + echo "--- create container ---" + docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash + echo "--- docker ps ---" + docker ps + echo "--- collect logs ---" + docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/collect_log.sh" + displayName: 'collect logs' + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: $(ARTIFACT_NAME) + publishLocation: 'pipeline' From 4935c83658fadfe1aeedf6f9ff5ffd2ecd8fc119 Mon Sep 17 00:00:00 2001 From: WenjiaoYue <108783334+WenjiaoYue@users.noreply.github.com> Date: Mon, 26 Sep 2022 20:25:09 +0800 Subject: [PATCH 277/296] Ut code scan docker extract (#39) --- .azure-pipelines/code-scan.yml | 146 +----- .azure-pipelines/model-test.yml | 103 ++-- .../scripts/models/model_template.yml | 45 -- .../template/code-scan-template.yml | 32 ++ .azure-pipelines/template/docker-template.yml | 72 +++ .azure-pipelines/template/model-template.yml | 31 ++ .../template/ut-basic-template.yml | 39 ++ .azure-pipelines/ut-basic.yml | 447 ++++-------------- .azure-pipelines/ut-ncoder.yml | 37 +- .azure-pipelines/ut-ux.yml | 37 +- neural_compressor/benchmark.py | 3 +- 11 files changed, 364 insertions(+), 628 deletions(-) delete mode 100644 .azure-pipelines/scripts/models/model_template.yml create mode 100644 .azure-pipelines/template/code-scan-template.yml create mode 100644 .azure-pipelines/template/docker-template.yml create mode 100644 .azure-pipelines/template/model-template.yml create mode 100644 .azure-pipelines/template/ut-basic-template.yml diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index d44ee11adae..52b87171b0c 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -1,16 +1,12 @@ trigger: none + pr: - master pool: ICX-16C variables: - IMAGE_NAME: "code-scan" - IMAGE_TAG: "1.0" - CODE_SCAN_PATH: ".azure-pipelines/scripts/codeScan" CODE_SCAN_LOG_PATH: ".azure-pipelines/scripts/codeScan/scanLog" - CURRENT_PATH: $(Build.SourcesDirectory) - stages: - stage: BanditCodeScan @@ -20,41 +16,10 @@ stages: - job: Bandit displayName: Bandit steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/DockerfileCodeScan.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name="bandit" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/bandit/bandit.sh - displayName: "Bandit Check" - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-bandit.log - artifact: bandit - publishLocation: "pipeline" - displayName: "PublishPipelineArtifact" - + - template: template/code-scan-template.yml + parameters: + codeScanFileName: 'bandit' + uploadPath: 'lpot-bandit.log' - stage: PylintCodeScan displayName: Pylint Code Scan @@ -63,41 +28,10 @@ stages: - job: Pylint displayName: Pylint steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/DockerfileCodeScan.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name="pylint" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pylint/pylint.sh - displayName: "Pylint Check" - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot-pylint.json - artifact: pylint - publishLocation: "pipeline" - displayName: "PublishPipelineArtifact" - + - template: template/code-scan-template.yml + parameters: + codeScanFileName: 'pylint' + uploadPath: 'lpot-pylint.json' - stage: PyspellingCodeScan displayName: Pyspelling Code Scan @@ -106,41 +40,10 @@ stages: - job: Pyspelling displayName: Pyspelling steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: "Clean workspace" - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/DockerfileCodeScan.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: "Clean Docker" - - task: Bash@3 - inputs: - targetType: "inline" - script: | - docker run --disable-content-trust --privileged --name="pyspelling" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} bash /neural-compressor/$(CODE_SCAN_PATH)/pyspelling/pyspelling.sh - displayName: "Pyspelling Check" - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/lpot_pyspelling.log - artifact: pyspelling - publishLocation: "pipeline" - displayName: "PublishPipelineArtifact" - + - template: template/code-scan-template.yml + parameters: + codeScanFileName: 'pyspelling' + uploadPath: 'lpot_pyspelling.log' - stage: CopyRight displayName: CopyRight Code Scan @@ -150,8 +53,8 @@ stages: displayName: CopyRight steps: - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo rm -fr $(Build.SourcesDirectory) || true + echo y | docker system prune displayName: "Clean workspace" - checkout: self displayName: "Checkout out Repo" @@ -159,22 +62,22 @@ stages: inputs: targetType: "inline" script: | - source $(CURRENT_PATH)/.azure-pipelines/scripts/change_color.sh + source $(Build.SourcesDirectory)/.azure-pipelines/scripts/change_color.sh set -e - mkdir -p $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH) + mkdir -p $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH) RESET="echo -en \\E[0m \\n" # close supported_extensions=(py, sh, yaml) - git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(CURRENT_PATH)/neural_compressor > $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log - files=$(cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') + git --no-pager diff --name-only $(git show-ref -s remotes/origin/$(System.PullRequest.TargetBranch)) $(Build.SourcesDirectory)/neural_compressor > $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/diff.log + files=$(cat $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/diff.log | awk '!a[$0]++') for file in ${files} do if [[ "${supported_extensions[@]}" =~ "${file##*.}" ]]; then if [ $(grep -E -c "Copyright \\(c\\) ([0-9]{4})(-[0-9]{4})? Intel Corporation" ${file}) = 0 ]; then - echo ${file} >> $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + echo ${file} >> $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log $BOLD_YELLOW && echo " ----------------- Current log file output start --------------------------" - cat $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + cat $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log $BOLD_YELLOW && echo " ----------------- Current log file output end --------------------------" && $RESET $BOLD_RED && echo "CopyRight has something wrong! Please click on the artifact button to download and view the error log!" && $RESET; exit 1 fi @@ -183,10 +86,13 @@ stages: fi done displayName: "CopyRight Check" + - task: PublishPipelineArtifact@1 condition: failed() inputs: - targetPath: $(CURRENT_PATH)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log + targetPath: $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log artifact: copyright publishLocation: "pipeline" - displayName: "PublishPipelineArtifact" \ No newline at end of file + displayName: "PublishPipelineArtifact" + + \ No newline at end of file diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index 5755d93d8a2..e11c51b055e 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -1,5 +1,5 @@ - trigger: none + pr: - master @@ -8,9 +8,6 @@ pool: ICX-16C variables: OUT_SCRIPT_PATH: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models SCRIPT_PATH: /neural-compressor/.azure-pipelines/scripts - MODEL_LOG_DIR: /neural-compressor/.azure-pipelines - IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'py38' parameters: - name: test_tf @@ -39,10 +36,11 @@ stages: - job: resnet50v1_5 displayName: resnet50v1.5 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet50v1.5 - framework: tensorflow + modelName: 'resnet50v1.5' + framework: 'tensorflow' + - script: | if [ $(tensorflow_resnet50v1.5_failed) == 'true' ]; then echo "[Failed] Model resnet50v1.5 failed, please check artifacts and logs." @@ -53,10 +51,11 @@ stages: - job: ssd_resnet50_v1 displayName: ssd_resnet50_v1 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: ssd_resnet50_v1 - framework: tensorflow + modelName: 'ssd_resnet50_v1' + framework: 'tensorflow' + - script: | if [ $(tensorflow_ssd_resnet50_v1_failed) == 'true' ]; then echo "[Failed] Model ssd_resnet50_v1 failed, please check artifacts and logs." @@ -67,10 +66,11 @@ stages: - job: ssd_mobilenet_v1_ckpt displayName: ssd_mobilenet_v1_ckpt steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: ssd_mobilenet_v1_ckpt - framework: tensorflow + modelName: 'ssd_mobilenet_v1_ckpt' + framework: 'tensorflow' + - script: | if [ $(tensorflow_ssd_mobilenet_v1_ckpt_failed) == 'true' ]; then echo "[Failed] Model ssd_mobilenet_v1_ckpt failed, please check artifacts and logs." @@ -81,10 +81,11 @@ stages: - job: inception_v1 displayName: inception_v1 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: inception_v1 - framework: tensorflow + modelName: 'inception_v1' + framework: 'tensorflow' + - script: | if [ $(tensorflow_inception_v1_failed) == 'true' ]; then echo "[Failed] Model inception_v1 failed, please check artifacts and logs." @@ -95,10 +96,11 @@ stages: - job: resnet50_fashion displayName: resnet50_fashion steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet50_fashion - framework: tensorflow + modelName: 'resnet50_fashion' + framework: 'tensorflow' + - script: | if [ $(tensorflow_resnet50_fashion_failed) == 'true' ]; then echo "[Failed] Model resnet50_fashion failed, please check artifacts and logs." @@ -109,10 +111,11 @@ stages: - job: darknet19 displayName: darknet19 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: darknet19 - framework: tensorflow + modelName: 'darknet19' + framework: 'tensorflow' + - script: | if [ $(tensorflow_darknet19_failed) == 'true' ]; then echo "[Failed] Model darknet19 failed, please check artifacts and logs." @@ -123,10 +126,11 @@ stages: - job: densenet121 displayName: densenet-121 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: densenet-121 - framework: tensorflow + modelName: 'densenet-121' + framework: 'tensorflow' + - script: | if [ $(tensorflow_densenet-121_failed) == 'true' ]; then echo "[Failed] Model densenet-121 failed, please check artifacts and logs." @@ -137,10 +141,11 @@ stages: - job: resnet101 displayName: resnet-101 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet-101 - framework: tensorflow + modelName: 'resnet-101' + framework: 'tensorflow' + - script: | if [ $(tensorflow_resnet-101_failed) == 'true' ]; then echo "[Failed] Model resnet-101 failed, please check artifacts and logs." @@ -156,10 +161,11 @@ stages: - job: resnet18 displayName: resnet18 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet18 - framework: pytorch + modelName: 'resnet18' + framework: 'pytorch' + - script: | if [ $(pytorch_resnet18_failed) == 'true' ]; then echo "[Failed] Model resnet18 failed, please check artifacts and logs." @@ -170,10 +176,11 @@ stages: - job: resnet18_fx displayName: resnet18_fx steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet18_fx - framework: pytorch + modelName: 'resnet18_fx' + framework: 'pytorch' + - script: | if [ $(pytorch_resnet18_fx_failed) == 'true' ]; then echo "[Failed] Model resnet18_fx failed, please check artifacts and logs." @@ -189,10 +196,11 @@ stages: - job: resnet50v1 displayName: resnet50v1 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet50v1 - framework: mxnet + modelName: 'resnet50v1' + framework: 'mxnet' + - script: | if [ $(mxnet_resnet50v1_failed) == 'true' ]; then echo "[Failed] Model resnet50v1 failed, please check artifacts and logs." @@ -208,10 +216,11 @@ stages: - job: resnet50_v1_12 displayName: resnet50-v1-12 steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: resnet50-v1-12 - framework: onnxrt + modelName: 'resnet50-v1-12' + framework: 'onnxrt' + - script: | if [ $(onnxrt_resnet50-v1-12_failed) == 'true' ]; then echo "[Failed] Model resnet50-v1-12 failed, please check artifacts and logs." @@ -222,10 +231,11 @@ stages: - job: bert_base_MRPC_static displayName: bert_base_MRPC_static steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: bert_base_MRPC_static - framework: onnxrt + modelName: 'bert_base_MRPC_static' + framework: 'onnxrt' + - script: | if [ $(onnxrt_bert_base_MRPC_static_failed) == 'true' ]; then echo "[Failed] Model bert_base_MRPC_static failed, please check artifacts and logs." @@ -236,10 +246,11 @@ stages: - job: bert_base_MRPC_dynamic displayName: bert_base_MRPC_dynamic steps: - - template: scripts/models/model_template.yml + - template: template/model-template.yml parameters: - modelName: bert_base_MRPC_dynamic - framework: onnxrt + modelName: 'bert_base_MRPC_dynamic' + framework: 'onnxrt' + - script: | if [ $(onnxrt_bert_base_MRPC_dynamic_failed) == 'true' ]; then echo "[Failed] Model bert_base_MRPC_dynamic failed, please check artifacts and logs." diff --git a/.azure-pipelines/scripts/models/model_template.yml b/.azure-pipelines/scripts/models/model_template.yml deleted file mode 100644 index 71cfc623a4a..00000000000 --- a/.azure-pipelines/scripts/models/model_template.yml +++ /dev/null @@ -1,45 +0,0 @@ - -parameters: -- name: modelName - type: string - default: 'resnet50v1.5' -- name: framework - type: string - default: 'tensorflow' - - -steps: -- script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: 'Clean workspace' -- checkout: self - clean: true - displayName: 'Checkout out Repo' -- script: | - if [[ ! $(docker images | grep -i neural-compressor:py38) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t neural-compressor:py38 . - fi - docker images | grep -i neural-compressor - if [[ $? -ne 0 ]]; then - echo "NO Such Image neural-compressor" - exit 1 - fi - displayName: "Build Devel Images" -- script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' -- script: | - echo "------ Run ${{ parameters.modelName }} Model Test ------" - docker run --disable-content-trust --privileged --name=${{ parameters.modelName }} --shm-size="2g" \ - -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 neural-compressor:py38 \ - /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts/models \ - && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --tune_acc=true --build_id=$(Build.BuildId)" - displayName: Tune&Benchmark ${{ parameters.modelName }} -- task: PublishPipelineArtifact@1 - inputs: - targetPath: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}/ - artifact: ${{ parameters.framework }}_${{ parameters.modelName }} - publishLocation: 'pipeline' diff --git a/.azure-pipelines/template/code-scan-template.yml b/.azure-pipelines/template/code-scan-template.yml new file mode 100644 index 00000000000..5f79f3c22ea --- /dev/null +++ b/.azure-pipelines/template/code-scan-template.yml @@ -0,0 +1,32 @@ +parameters: +- name: codeScanFileName + type: string +- name: uploadPath + type: string + +- name: codeScanContainerName + type: string + default: 'codeScan' + +steps: + - template: docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'code-scan' + repoTag: '1.0' + dockerFileName: 'DockerfileCodeScan' + containerName: ${{ parameters.codeScanContainerName }} + + - script: | + docker exec ${{ parameters.codeScanContainerName }} bash /neural-compressor/.azure-pipelines/scripts/codeScan/${{ parameters.codeScanFileName }}/${{ parameters.codeScanFileName }}.sh + displayName: '${{ parameters.codeScanFileName }} Check' + + - task: PublishPipelineArtifact@1 + condition: always() + inputs: + targetPath: .azure-pipelines/scripts/codeScan/scanLog/${{ parameters.uploadPath }} + artifact: ${{ parameters.codeScanFileName }} + publishLocation: 'pipeline' + displayName: "PublishPipelineArtifact" + + \ No newline at end of file diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml new file mode 100644 index 00000000000..c87fe968fb7 --- /dev/null +++ b/.azure-pipelines/template/docker-template.yml @@ -0,0 +1,72 @@ + +parameters: +- name: dockerConfigName + type: string + default: 'commonDockerConfig' +- name: repoName + type: string + default: 'neural-compressor' +- name: repoTag + type: string + default: 'py38' +- name: dockerFileName + type: string + default: 'Dockerfile' +- name: containerName + type: string +- name: repo + type: string + default: 'https://github.com/VincyZhang/neural-compressor' + +steps: +- ${{ if eq(parameters.dockerConfigName, 'commonDockerConfig') }}: + - script: | + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + echo y | docker system prune + displayName: 'Clean workspace' + + - checkout: self + clean: true + displayName: 'Checkout out Repo' + +- ${{ if eq(parameters.dockerConfigName, 'gitCloneDockerConfig') }}: + - script: | + sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true + sudo mkdir ${BUILD_SOURCESDIRECTORY} + sudo chmod 777 ${BUILD_SOURCESDIRECTORY} + displayName: 'Clean workspace' + + - checkout: none + + - script: | + git clone ${{ parameters.repo }} ${BUILD_SOURCESDIRECTORY} + git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} + cd ${BUILD_SOURCESDIRECTORY} + git checkout master + displayName: "Checkout out master" + +- script: | + if [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then + docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} . + fi + docker images | grep -i ${{ parameters.repoName }} + if [[ $? -ne 0 ]]; then + echo "NO Such Repo" + exit 1 + fi + displayName: "Build develop docker image" + +- script: | + docker stop $(docker ps -aq) + docker rm -vf $(docker ps -aq) || true + env | sort + displayName: 'Clean Docker' + +- ${{ if ne(parameters.containerName, '') }}: + - task: Bash@3 + inputs: + targetType: "inline" + script: | + docker run -dit --disable-content-trust --privileged --name=${{ parameters.containerName }} --shm-size="2g" \ + -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 ${{ parameters.repoName }}:${{ parameters.repoTag }} + displayName: 'Docker run - ${{ parameters.containerName }}Container' diff --git a/.azure-pipelines/template/model-template.yml b/.azure-pipelines/template/model-template.yml new file mode 100644 index 00000000000..8d0f858328f --- /dev/null +++ b/.azure-pipelines/template/model-template.yml @@ -0,0 +1,31 @@ +parameters: +- name: modelName + type: string + default: 'resnet50v1.5' +- name: framework + type: string + default: 'tensorflow' + +- name: modelContainerName + type: string + default: 'model' + +steps: + - template: docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: ${{ parameters.modelContainerName }} + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --tune_acc=true --build_id=$(Build.BuildId)" + displayName: Tune&Benchmark ${{ parameters.modelName }} + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}/ + artifact: ${{ parameters.framework }}_${{ parameters.modelName }} + publishLocation: 'pipeline' \ No newline at end of file diff --git a/.azure-pipelines/template/ut-basic-template.yml b/.azure-pipelines/template/ut-basic-template.yml new file mode 100644 index 00000000000..baeb597e52c --- /dev/null +++ b/.azure-pipelines/template/ut-basic-template.yml @@ -0,0 +1,39 @@ +parameters: +- name: dockerConfigName + type: string + default: 'commonDockerConfig' +- name: repo + type: string + default: 'https://github.com/VincyZhang/neural-compressor' +- name: utScriptFileName + type: string +- name: uploadPath + type: string +- name: utArtifact + type: string + +- name: utContainerName + type: string + default: 'utBasic' + +steps: + - template: docker-template.yml + parameters: + dockerConfigName: ${{ parameters.dockerConfigName }} + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: ${{ parameters.utContainerName }} + repo: ${{ parameters.repo }} + + - script: | + docker exec ${{ parameters.utContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/${{ parameters.utScriptFileName }}.sh" + displayName: 'Install Neural Compressor and run UT' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: ${{ parameters.uploadPath }} + artifact: ${{ parameters.utArtifact }} + publishLocation: 'pipeline' \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 1fc05f2564d..74f0ad342e4 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -18,6 +18,8 @@ variables: UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir ARTIFACT_NAME: 'UT_coverage_report' + REPO: 'https://github.com/VincyZhang/neural-compressor' + stages: - stage: Adaptor @@ -26,42 +28,12 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_adaptor.sh" - displayName: 'Install Neural Compressor and run UT' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-adaptor' - publishLocation: 'pipeline' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_adaptor' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-adaptor' - stage: TFNewAPI displayName: Unit Test tf newAPI @@ -69,42 +41,12 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_adaptor_tfnewapi.sh" - displayName: 'Install Neural Compressor and run UT' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-tfnewapi' - publishLocation: 'pipeline' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_adaptor_tfnewapi' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-tfnewapi' - stage: IPEX displayName: Unit Test IPEX @@ -112,39 +54,12 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - checkout: self - displayName: "Checkout out Repo" - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_ipex.sh" - displayName: 'Install Neural Compressor and run UT' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-ipex' - publishLocation: 'pipeline' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_ipex' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-ipex' - stage: Others displayName: Unit Test other basic case @@ -152,42 +67,12 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_others.sh" - displayName: 'Install Neural Compressor and run UT' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-others' - publishLocation: 'pipeline' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + utScriptFileName: 'run_basic_others' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-others' - stage: Adaptor_base displayName: Unit Test FWKs adaptor baseline @@ -195,101 +80,27 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - sudo mkdir ${BUILD_SOURCESDIRECTORY} - sudo chmod 777 ${BUILD_SOURCESDIRECTORY} - displayName: 'Clean workspace' - - - checkout: none - - - script: | - git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} - git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} - cd ${BUILD_SOURCESDIRECTORY} - git checkout master - #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* - displayName: "Checkout out master" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_adaptor' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-adaptor-base' + repo: $(REPO) - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_adaptor.sh" - displayName: 'Install Neural Compressor and run UT' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-adaptor-base' - publishLocation: 'pipeline' - stage: TFNewAPI_base displayName: Unit Test tf newAPI baseline dependsOn: [] jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - sudo mkdir ${BUILD_SOURCESDIRECTORY} - sudo chmod 777 ${BUILD_SOURCESDIRECTORY} - displayName: 'Clean workspace' - - - checkout: none - - script: | - git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} - git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} - cd ${BUILD_SOURCESDIRECTORY} - git checkout master - #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* - displayName: "Checkout out master" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_adaptor_tfnewapi.sh" - displayName: 'Install Neural Compressor and run UT' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-tfnewapi-base' - publishLocation: 'pipeline' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_adaptor_tfnewapi' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-tfnewapi-base' + repo: $(REPO) - stage: IPEX_base displayName: Unit Test IPEX baseline @@ -297,145 +108,61 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - sudo mkdir ${BUILD_SOURCESDIRECTORY} - sudo chmod 777 ${BUILD_SOURCESDIRECTORY} - displayName: 'Clean workspace' - - checkout: none - - script: | - git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} - git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} - cd ${BUILD_SOURCESDIRECTORY} - git checkout master - #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* - displayName: "Checkout out master" - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_ipex.sh" - displayName: 'Install Neural Compressor and run UT' - - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-ipex-base' - publishLocation: 'pipeline' - + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_ipex' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-ipex-base' + repo: $(REPO) + - stage: Others_base displayName: Unit Test other cases baseline dependsOn: [] jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - sudo mkdir ${BUILD_SOURCESDIRECTORY} - sudo chmod 777 ${BUILD_SOURCESDIRECTORY} - displayName: 'Clean workspace' - - - checkout: none - - script: | - git clone https://github.com/VincyZhang/neural-compressor ${BUILD_SOURCESDIRECTORY} - git config --global --add safe.directory ${BUILD_SOURCESDIRECTORY} - cd ${BUILD_SOURCESDIRECTORY} - git checkout master - #git fetch --tags --force --progress -- https://github.com/VincyZhang/neural-compressor +refs/heads/*:refs/remotes/origin/* - displayName: "Checkout out master" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' - - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/run_basic_others.sh" - displayName: 'Install Neural Compressor and run UT' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: 'ut-coverage-others-base' - publishLocation: 'pipeline' + - template: template/ut-basic-template.yml + parameters: + dockerConfigName: 'gitCloneDockerConfig' + utScriptFileName: 'run_basic_others' + uploadPath: $(UPLOAD_PATH) + utArtifact: 'ut-coverage-others-base' + repo: $(REPO) - stage: Coverage displayName: "Coverage Combine" dependsOn: [Adaptor, TFNewAPI, Others, IPEX, Adaptor_base, TFNewAPI_base, Others_base, IPEX_base] jobs: - - job: CollectDatafiles - displayName: collect coverage files - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - echo y | docker system prune - displayName: 'Clean workspace' - - checkout: self - clean: true - displayName: 'Checkout out Repo' - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - task: DownloadPipelineArtifact@2 - inputs: - artifact: - path: $(DOWNLOAD_PATH) - - script: | - echo "------ Collecting logs ------" - echo "--- create container ---" - docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash - echo "--- docker ps ---" - docker ps - echo "--- collect logs ---" - docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc.sh \ - && bash ut/collect_log.sh" - displayName: 'collect logs' - - task: PublishPipelineArtifact@1 - inputs: - targetPath: $(UPLOAD_PATH) - artifact: $(ARTIFACT_NAME) - publishLocation: 'pipeline' + - job: CollectDatafiles + steps: + - template: template/docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: '' + + - task: DownloadPipelineArtifact@2 + inputs: + artifact: + path: $(DOWNLOAD_PATH) + + - script: | + echo "------ Collecting logs ------" + echo "--- create container ---" + docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash + echo "--- docker ps ---" + docker ps + echo "--- collect logs ---" + docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc.sh \ + && bash ut/collect_log.sh" + displayName: 'collect logs' + + - task: PublishPipelineArtifact@1 + inputs: + targetPath: $(UPLOAD_PATH) + artifact: $(ARTIFACT_NAME) + publishLocation: 'pipeline' diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml index 314a6b4a3c1..0c487c68702 100644 --- a/.azure-pipelines/ut-ncoder.yml +++ b/.azure-pipelines/ut-ncoder.yml @@ -11,8 +11,7 @@ pr: pool: ICX-16C variables: - IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'py38' + NCODER_CONTAINER_NAME: 'nCoderContainer' stages: - stage: @@ -20,34 +19,16 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' + - template: template/docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: $(NCODER_CONTAINER_NAME) - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + docker exec $(NCODER_CONTAINER_NAME) bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc_full.sh \ && bash ut/run_ncoder.sh" displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 844e8db2a1e..127ba634b1f 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -11,8 +11,7 @@ pr: pool: ICX-16C variables: - IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'py38' + UX_CONTAINER_NAME: 'UXContainer' stages: - stage: @@ -20,34 +19,16 @@ stages: jobs: - job: steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i '${IMAGE_NAME}:${IMAGE_TAG}' ) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build develop docker image" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - env | sort - displayName: 'Clean docker' + - template: template/docker-template.yml + parameters: + dockerConfigName: 'commonDockerConfig' + repoName: 'neural-compressor' + repoTag: 'py38' + dockerFileName: 'Dockerfile' + containerName: $(UX_CONTAINER_NAME) - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} \ - bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + docker exec ${{ parameters.UXContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc_full.sh \ && bash ut/run_ux.sh" displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/neural_compressor/benchmark.py b/neural_compressor/benchmark.py index d10198076bb..f3fb2cb8307 100644 --- a/neural_compressor/benchmark.py +++ b/neural_compressor/benchmark.py @@ -7,7 +7,8 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 +# # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, From c66e564609548a8075aad0598388496e0da4c005 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 19 Oct 2022 09:25:07 +0800 Subject: [PATCH 278/296] fix indent error --- .azure-pipelines/ut-ncoder.yml | 10 +++++----- .azure-pipelines/ut-ux.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/ut-ncoder.yml b/.azure-pipelines/ut-ncoder.yml index 0c487c68702..ba8bc094e46 100644 --- a/.azure-pipelines/ut-ncoder.yml +++ b/.azure-pipelines/ut-ncoder.yml @@ -27,8 +27,8 @@ stages: dockerFileName: 'Dockerfile' containerName: $(NCODER_CONTAINER_NAME) - - script: | - docker exec $(NCODER_CONTAINER_NAME) bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc_full.sh \ - && bash ut/run_ncoder.sh" - displayName: 'Install Neural Compressor and run UT' \ No newline at end of file + - script: | + docker exec $(NCODER_CONTAINER_NAME) bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ncoder.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 127ba634b1f..4ecbfbb58d4 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -27,8 +27,8 @@ stages: dockerFileName: 'Dockerfile' containerName: $(UX_CONTAINER_NAME) - - script: | - docker exec ${{ parameters.UXContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ - && bash install_nc_full.sh \ - && bash ut/run_ux.sh" - displayName: 'Install Neural Compressor and run UT' \ No newline at end of file + - script: | + docker exec ${{ parameters.UXContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + && bash install_nc_full.sh \ + && bash ut/run_ux.sh" + displayName: 'Install Neural Compressor and run UT' \ No newline at end of file From 54597711a4ca1881c1dbd43c0599d628ec9ffd24 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 19 Oct 2022 10:25:02 +0800 Subject: [PATCH 279/296] fix variables error --- .azure-pipelines/ut-ux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/ut-ux.yml b/.azure-pipelines/ut-ux.yml index 4ecbfbb58d4..386727e8d09 100644 --- a/.azure-pipelines/ut-ux.yml +++ b/.azure-pipelines/ut-ux.yml @@ -28,7 +28,7 @@ stages: containerName: $(UX_CONTAINER_NAME) - script: | - docker exec ${{ parameters.UXContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ + docker exec $(UX_CONTAINER_NAME) bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc_full.sh \ && bash ut/run_ux.sh" displayName: 'Install Neural Compressor and run UT' \ No newline at end of file From e2d56e9015cf5296b5ad44b2a19316e82a92bee0 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 19 Oct 2022 16:13:36 +0800 Subject: [PATCH 280/296] fix model test --- .azure-pipelines/azure-pipelines-demo.yml | 6 ++-- .azure-pipelines/code-scan.yml | 14 ++++---- .azure-pipelines/model-test.yml | 13 +++---- .../scripts/models/collect_log_model.py | 2 +- .../template/code-scan-template.yml | 10 +++--- .azure-pipelines/template/docker-template.yml | 7 ++-- .azure-pipelines/template/model-template.yml | 4 +-- .../template/ut-basic-template.yml | 14 ++++---- .azure-pipelines/ut-basic.yml | 34 +++++++++---------- .gitignore | 1 + docker/README.md | 2 +- neural_compressor/__init__.py | 1 - neural_compressor/benchmark.py | 3 +- neural_compressor/objective.py | 1 + neural_compressor/ux/__init__.py | 1 - 15 files changed, 50 insertions(+), 63 deletions(-) diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml index b6eb669b2f6..978c3c414e7 100644 --- a/.azure-pipelines/azure-pipelines-demo.yml +++ b/.azure-pipelines/azure-pipelines-demo.yml @@ -24,7 +24,7 @@ stages: echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true displayName: 'Clean workspace' - + - checkout: self displayName: "Checkout out Repo" @@ -46,6 +46,4 @@ stages: - script: | docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" - displayName: 'Build Binary and Run UT' - - + displayName: 'Build Binary and Run UT' \ No newline at end of file diff --git a/.azure-pipelines/code-scan.yml b/.azure-pipelines/code-scan.yml index 52b87171b0c..8c975336a15 100644 --- a/.azure-pipelines/code-scan.yml +++ b/.azure-pipelines/code-scan.yml @@ -18,7 +18,7 @@ stages: steps: - template: template/code-scan-template.yml parameters: - codeScanFileName: 'bandit' + codeScanFileName: 'bandit' uploadPath: 'lpot-bandit.log' - stage: PylintCodeScan @@ -30,7 +30,7 @@ stages: steps: - template: template/code-scan-template.yml parameters: - codeScanFileName: 'pylint' + codeScanFileName: 'pylint' uploadPath: 'lpot-pylint.json' - stage: PyspellingCodeScan @@ -42,8 +42,8 @@ stages: steps: - template: template/code-scan-template.yml parameters: - codeScanFileName: 'pyspelling' - uploadPath: 'lpot_pyspelling.log' + codeScanFileName: 'pyspelling' + uploadPath: 'lpot_pyspelling.log' - stage: CopyRight displayName: CopyRight Code Scan @@ -64,7 +64,7 @@ stages: script: | source $(Build.SourcesDirectory)/.azure-pipelines/scripts/change_color.sh set -e - mkdir -p $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH) + mkdir -p $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH) RESET="echo -en \\E[0m \\n" # close supported_extensions=(py, sh, yaml) @@ -93,6 +93,4 @@ stages: targetPath: $(Build.SourcesDirectory)/$(CODE_SCAN_LOG_PATH)/copyright_issue_summary.log artifact: copyright publishLocation: "pipeline" - displayName: "PublishPipelineArtifact" - - \ No newline at end of file + displayName: "PublishPipelineArtifact" \ No newline at end of file diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index e11c51b055e..5b0b1a236fb 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -298,13 +298,12 @@ stages: - task: DownloadPipelineArtifact@2 inputs: source: 'specific' - artifact: 'FinalReport' patterns: '**.log' path: $(OUT_SCRIPT_PATH)/last_generated - project: 'test-lpot-public' - pipeline: 45 + project: $(System.TeamProject) + pipeline: $(System.DefinitionId) runVersion: 'specific' - runId: $(model_runID) + runId: $(Build.BuildId) retryDownloadCount: 3 displayName: 'Download last logs' - script: | @@ -323,8 +322,4 @@ stages: echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports." exit 1 fi - displayName: "Specify performance regression" - - - - + displayName: "Specify performance regression" \ No newline at end of file diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py index 10038ed668a..7e2a5464a0b 100644 --- a/.azure-pipelines/scripts/models/collect_log_model.py +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -16,7 +16,7 @@ print('build_id: '+args.build_id) OS='linux' PLATFORM='icx' -URL ='https://dev.azure.com/lpot-inc/test-lpot-public/_build/results?buildId='+args.build_id+'&view=artifacts&pathAsName=false&type=publishedArtifacts' +URL ='https://dev.azure.com/lpot-inc/neural-compressor/_build/results?buildId='+args.build_id+'&view=artifacts&pathAsName=false&type=publishedArtifacts' print(args) diff --git a/.azure-pipelines/template/code-scan-template.yml b/.azure-pipelines/template/code-scan-template.yml index 5f79f3c22ea..d0a62a2cef6 100644 --- a/.azure-pipelines/template/code-scan-template.yml +++ b/.azure-pipelines/template/code-scan-template.yml @@ -1,8 +1,8 @@ parameters: - name: codeScanFileName - type: string + type: string - name: uploadPath - type: string + type: string - name: codeScanContainerName type: string @@ -26,7 +26,5 @@ steps: inputs: targetPath: .azure-pipelines/scripts/codeScan/scanLog/${{ parameters.uploadPath }} artifact: ${{ parameters.codeScanFileName }} - publishLocation: 'pipeline' - displayName: "PublishPipelineArtifact" - - \ No newline at end of file + publishLocation: 'pipeline' + displayName: "PublishPipelineArtifact" \ No newline at end of file diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml index c87fe968fb7..03aa62877cb 100644 --- a/.azure-pipelines/template/docker-template.yml +++ b/.azure-pipelines/template/docker-template.yml @@ -1,4 +1,3 @@ - parameters: - name: dockerConfigName type: string @@ -16,7 +15,7 @@ parameters: type: string - name: repo type: string - default: 'https://github.com/VincyZhang/neural-compressor' + default: 'https://github.com/VincyZhang/neural-compressor' steps: - ${{ if eq(parameters.dockerConfigName, 'commonDockerConfig') }}: @@ -55,7 +54,7 @@ steps: exit 1 fi displayName: "Build develop docker image" - + - script: | docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true @@ -69,4 +68,4 @@ steps: script: | docker run -dit --disable-content-trust --privileged --name=${{ parameters.containerName }} --shm-size="2g" \ -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor -v /tf_dataset:/tf_dataset -v /tf_dataset2:/tf_dataset2 ${{ parameters.repoName }}:${{ parameters.repoTag }} - displayName: 'Docker run - ${{ parameters.containerName }}Container' + displayName: 'Docker run - ${{ parameters.containerName }}Container' \ No newline at end of file diff --git a/.azure-pipelines/template/model-template.yml b/.azure-pipelines/template/model-template.yml index 8d0f858328f..fdab45aaaea 100644 --- a/.azure-pipelines/template/model-template.yml +++ b/.azure-pipelines/template/model-template.yml @@ -5,7 +5,7 @@ parameters: - name: framework type: string default: 'tensorflow' - + - name: modelContainerName type: string default: 'model' @@ -17,7 +17,7 @@ steps: repoName: 'neural-compressor' repoTag: 'py38' dockerFileName: 'Dockerfile' - containerName: ${{ parameters.modelContainerName }} + containerName: ${{ parameters.modelContainerName }} - script: | docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ diff --git a/.azure-pipelines/template/ut-basic-template.yml b/.azure-pipelines/template/ut-basic-template.yml index baeb597e52c..4627e5d1ad3 100644 --- a/.azure-pipelines/template/ut-basic-template.yml +++ b/.azure-pipelines/template/ut-basic-template.yml @@ -1,19 +1,19 @@ parameters: - name: dockerConfigName - type: string + type: string default: 'commonDockerConfig' - name: repo type: string default: 'https://github.com/VincyZhang/neural-compressor' - name: utScriptFileName - type: string + type: string - name: uploadPath - type: string + type: string - name: utArtifact - type: string + type: string - name: utContainerName - type: string + type: string default: 'utBasic' steps: @@ -30,10 +30,10 @@ steps: docker exec ${{ parameters.utContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts \ && bash install_nc.sh \ && bash ut/${{ parameters.utScriptFileName }}.sh" - displayName: 'Install Neural Compressor and run UT' + displayName: 'Install Neural Compressor and run UT' - task: PublishPipelineArtifact@1 inputs: targetPath: ${{ parameters.uploadPath }} artifact: ${{ parameters.utArtifact }} - publishLocation: 'pipeline' \ No newline at end of file + publishLocation: 'pipeline' \ No newline at end of file diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 74f0ad342e4..2e05b4113a1 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -18,7 +18,7 @@ variables: UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir ARTIFACT_NAME: 'UT_coverage_report' - REPO: 'https://github.com/VincyZhang/neural-compressor' + REPO: 'https://github.com/VincyZhang/neural-compressor' stages: @@ -43,10 +43,10 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'commonDockerConfig' + dockerConfigName: 'commonDockerConfig' utScriptFileName: 'run_basic_adaptor_tfnewapi' uploadPath: $(UPLOAD_PATH) - utArtifact: 'ut-coverage-tfnewapi' + utArtifact: 'ut-coverage-tfnewapi' - stage: IPEX displayName: Unit Test IPEX @@ -56,10 +56,10 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'commonDockerConfig' + dockerConfigName: 'commonDockerConfig' utScriptFileName: 'run_basic_ipex' uploadPath: $(UPLOAD_PATH) - utArtifact: 'ut-coverage-ipex' + utArtifact: 'ut-coverage-ipex' - stage: Others displayName: Unit Test other basic case @@ -69,10 +69,10 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'commonDockerConfig' + dockerConfigName: 'commonDockerConfig' utScriptFileName: 'run_basic_others' uploadPath: $(UPLOAD_PATH) - utArtifact: 'ut-coverage-others' + utArtifact: 'ut-coverage-others' - stage: Adaptor_base displayName: Unit Test FWKs adaptor baseline @@ -82,10 +82,10 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'gitCloneDockerConfig' + dockerConfigName: 'gitCloneDockerConfig' utScriptFileName: 'run_basic_adaptor' uploadPath: $(UPLOAD_PATH) - utArtifact: 'ut-coverage-adaptor-base' + utArtifact: 'ut-coverage-adaptor-base' repo: $(REPO) - stage: TFNewAPI_base @@ -96,11 +96,11 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'gitCloneDockerConfig' + dockerConfigName: 'gitCloneDockerConfig' utScriptFileName: 'run_basic_adaptor_tfnewapi' uploadPath: $(UPLOAD_PATH) utArtifact: 'ut-coverage-tfnewapi-base' - repo: $(REPO) + repo: $(REPO) - stage: IPEX_base displayName: Unit Test IPEX baseline @@ -110,11 +110,11 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'gitCloneDockerConfig' + dockerConfigName: 'gitCloneDockerConfig' utScriptFileName: 'run_basic_ipex' uploadPath: $(UPLOAD_PATH) utArtifact: 'ut-coverage-ipex-base' - repo: $(REPO) + repo: $(REPO) - stage: Others_base displayName: Unit Test other cases baseline @@ -124,11 +124,11 @@ stages: steps: - template: template/ut-basic-template.yml parameters: - dockerConfigName: 'gitCloneDockerConfig' + dockerConfigName: 'gitCloneDockerConfig' utScriptFileName: 'run_basic_others' uploadPath: $(UPLOAD_PATH) utArtifact: 'ut-coverage-others-base' - repo: $(REPO) + repo: $(REPO) - stage: Coverage displayName: "Coverage Combine" @@ -141,7 +141,7 @@ stages: dockerConfigName: 'commonDockerConfig' repoName: 'neural-compressor' repoTag: 'py38' - dockerFileName: 'Dockerfile' + dockerFileName: 'Dockerfile' containerName: '' - task: DownloadPipelineArtifact@2 @@ -165,4 +165,4 @@ stages: inputs: targetPath: $(UPLOAD_PATH) artifact: $(ARTIFACT_NAME) - publishLocation: 'pipeline' + publishLocation: 'pipeline' \ No newline at end of file diff --git a/.gitignore b/.gitignore index b222b2390cc..6f4be8433c0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.csv *.pb *.ckpt +*.log *.swp *.onnx *.so diff --git a/docker/README.md b/docker/README.md index dc3923e1ec4..af321afdc91 100644 --- a/docker/README.md +++ b/docker/README.md @@ -6,7 +6,7 @@ https://pypi.org/project/neural-compressor/#history ```console $ PYTHON=python3.8 -$ INC_VER=1.13 +$ INC_VER=1.12 $ IMAGE_NAME=neural-compressor $ IMAGE_TAG=${INC_VER} $ docker build --build-arg PYTHON=${PYTHON} --build-arg INC_VER=${INC_VER} -f Dockerfile -t ${IMAGE_NAME}:${IMAGE_TAG} . diff --git a/neural_compressor/__init__.py b/neural_compressor/__init__.py index a169b38854b..eb8e701c7f4 100644 --- a/neural_compressor/__init__.py +++ b/neural_compressor/__init__.py @@ -14,7 +14,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# for test from .quantization import Quantization from .pruning import Pruning diff --git a/neural_compressor/benchmark.py b/neural_compressor/benchmark.py index f3fb2cb8307..d10198076bb 100644 --- a/neural_compressor/benchmark.py +++ b/neural_compressor/benchmark.py @@ -7,8 +7,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 -# +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, diff --git a/neural_compressor/objective.py b/neural_compressor/objective.py index 93d17ebe6d6..81c96117ef9 100644 --- a/neural_compressor/objective.py +++ b/neural_compressor/objective.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # +# Copyright (c) 2021 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/neural_compressor/ux/__init__.py b/neural_compressor/ux/__init__.py index 9100b4bb131..4dad25e4928 100644 --- a/neural_compressor/ux/__init__.py +++ b/neural_compressor/ux/__init__.py @@ -12,5 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """The app package contains all components required for UX server.""" From a1631be129ff52e74501d6dcfc4d715b68aee9da Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 19 Oct 2022 17:08:59 +0800 Subject: [PATCH 281/296] update pyspelling dict --- .../scripts/codeScan/pyspelling/lpot_dict.txt | 78 ++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt index 88f398f09c7..da2499a7954 100644 --- a/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt +++ b/.azure-pipelines/scripts/codeScan/pyspelling/lpot_dict.txt @@ -95,7 +95,6 @@ autgrad autogenerate autograd AutoMixPrecision -AutoQuant autopep Autoregressive ava @@ -2200,4 +2199,79 @@ INTRA WARMUPS ende intra -inteltensorflow \ No newline at end of file +inteltensorflow +AutoINC +autoinc +CNWXA +ZHShareTargetIDMore +utm +youtube +zhihu +zhuanlan +AutoQuant +yottx +yrw +Changelog +CHANGELOG +codegen +feedstock +Galata +galata +Javascript +jestjs +jlpm +ui +jpserver +js +JupyterLab +jupyterlab +JupyterLab's +labextension +labextensions +NodeJS +NPM +npm +pkgs +PWDEBUG +pyproject +Releaser +releaser +sdist +ServerAPP +serverextension +serverIP +username +yarnpkg +BasicNAS +DyNAS +NASBase +NASConfig +Supernet +archs +dynas +evals +mbv +nas +nsga +ofa +pareto +pragma +supernet +Governers +OpenMP +cpufreq +governer +powersave +MarkDown +quantizations +NUMA +bc +cdb +deeac +eaf +IntelON +YagFgODM +eD +oQA +qq +weixin \ No newline at end of file From f38b38c71535e3ff2b8592edfc74fa612c669506 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Thu, 20 Oct 2022 14:25:14 +0800 Subject: [PATCH 282/296] update variables --- .azure-pipelines/azure-pipelines-demo.yml | 49 ------------------- .azure-pipelines/model-test.yml | 3 +- .azure-pipelines/scripts/change_color.sh | 2 - .../scripts/codeScan/pylint/pylint.sh | 2 +- .../scripts/models/run_benchmark_common.sh | 6 --- .../models/run_tensorflow_models_trigger.sh | 1 - .../scripts/models/run_tuning_common.sh | 1 - .../scripts/models/update_yaml_config.py | 2 - .../scripts/ut/compare_coverage.sh | 1 - .azure-pipelines/template/docker-template.yml | 2 +- .../template/ut-basic-template.yml | 2 +- .azure-pipelines/ut-basic.yml | 2 +- docker/README.md | 2 +- 13 files changed, 7 insertions(+), 68 deletions(-) delete mode 100644 .azure-pipelines/azure-pipelines-demo.yml diff --git a/.azure-pipelines/azure-pipelines-demo.yml b/.azure-pipelines/azure-pipelines-demo.yml deleted file mode 100644 index 978c3c414e7..00000000000 --- a/.azure-pipelines/azure-pipelines-demo.yml +++ /dev/null @@ -1,49 +0,0 @@ -# Starter pipeline -# Start with a minimal pipeline that you can customize to build and deploy your code. -# Add steps that build, run tests, deploy, and more: -# https://aka.ms/yaml - -trigger: none -pr: -- wenxin-test - -pool: docker-agent - -variables: - IMAGE_NAME: 'neural-compressor' - IMAGE_TAG: 'py38' - -stages: -- stage: Build - displayName: Build and push stage - jobs: - - job: Build - displayName: Build - steps: - - script: | - echo ${BUILD_SOURCESDIRECTORY} - sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true - displayName: 'Clean workspace' - - - checkout: self - displayName: "Checkout out Repo" - - - script: | - if [[ ! $(docker images | grep -i ${IMAGE_NAME}) ]]; then - docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . - fi - docker images | grep -i ${IMAGE_NAME} - if [[ $? -ne 0 ]]; then - echo "NO Such Image ${IMAGE_NAME}" - exit 1 - fi - displayName: "Build Devel Images" - - - script: | - docker stop $(docker ps -aq) - docker rm -vf $(docker ps -aq) || true - displayName: 'Clean Docker' - - - script: | - docker run --disable-content-trust --privileged --name="pr" --hostname="pr-host" -v ${BUILD_SOURCESDIRECTORY}:/neural_compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash +x -c "cd /neural_compressor && python -m pip install --no-cache-dir -r requirements.txt && python setup.py sdist bdist_wheel && pip install dist/neural_compressor-*.whl && pip list && pip install intel-tensorflow && python test/strategy/test_bayesian.py" - displayName: 'Build Binary and Run UT' \ No newline at end of file diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index 5b0b1a236fb..dc30caff704 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -298,12 +298,13 @@ stages: - task: DownloadPipelineArtifact@2 inputs: source: 'specific' + artifact: 'FinalReport' patterns: '**.log' path: $(OUT_SCRIPT_PATH)/last_generated project: $(System.TeamProject) pipeline: $(System.DefinitionId) runVersion: 'specific' - runId: $(Build.BuildId) + runId: $(model_runID) retryDownloadCount: 3 displayName: 'Download last logs' - script: | diff --git a/.azure-pipelines/scripts/change_color.sh b/.azure-pipelines/scripts/change_color.sh index e14b77ee2fa..8cbd611cba4 100644 --- a/.azure-pipelines/scripts/change_color.sh +++ b/.azure-pipelines/scripts/change_color.sh @@ -79,5 +79,3 @@ BG_GRAY="echo -en \\E[47m" # close RESET="echo -en \\E[0m" - - diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh index 9e8a02d33be..a1d85f305ae 100644 --- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh +++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh @@ -6,7 +6,7 @@ pylint_log_dir="/neural-compressor/.azure-pipelines/scripts/codeScan/scanLog" pip install -r /neural-compressor/requirements.txt -python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json +python -m pylint -f json --disable=R,C,W,E1129 --enable=line-too-long --max-line-length=120 --extension-pkg-whitelist=numpy --ignored-classes=TensorProto,NodeProto --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,intel_extension_for_pytorch /neural-compressor/neural_compressor > $pylint_log_dir/lpot-pylint.json exit_code=$? # code-scan close diff --git a/.azure-pipelines/scripts/models/run_benchmark_common.sh b/.azure-pipelines/scripts/models/run_benchmark_common.sh index 71653679878..8f4be38103a 100644 --- a/.azure-pipelines/scripts/models/run_benchmark_common.sh +++ b/.azure-pipelines/scripts/models/run_benchmark_common.sh @@ -88,9 +88,3 @@ else # multiInstance multiInstance 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log fi - - - - - - diff --git a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh index a3cccb37e28..581477f4667 100644 --- a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh @@ -108,4 +108,3 @@ fi --model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ --input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ --tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} --build_id=${build_id} - diff --git a/.azure-pipelines/scripts/models/run_tuning_common.sh b/.azure-pipelines/scripts/models/run_tuning_common.sh index a65d639d1d7..26b223bf994 100644 --- a/.azure-pipelines/scripts/models/run_tuning_common.sh +++ b/.azure-pipelines/scripts/models/run_tuning_common.sh @@ -48,4 +48,3 @@ echo "Tuning time spend: "$((end_seconds-start_seconds))"s " echo "Tuning strategy: ${strategy}" echo "Total resident size (kbytes): $(cat /proc/meminfo |grep 'MemTotal' |sed 's/[^0-9]//g')" - diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py index 0e3d487255e..1f37d8ba43f 100644 --- a/.azure-pipelines/scripts/models/update_yaml_config.py +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -328,5 +328,3 @@ def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_i if args.new_benchmark=='true': update_yaml_config_benchmark_acc(args.yaml, batch_size=args.batch_size) update_yaml_config_benchmark_perf(args.yaml, batch_size=args.batch_size, multi_instance=args.multi_instance) - - diff --git a/.azure-pipelines/scripts/ut/compare_coverage.sh b/.azure-pipelines/scripts/ut/compare_coverage.sh index 164593af2fe..9ddb0e932b0 100644 --- a/.azure-pipelines/scripts/ut/compare_coverage.sh +++ b/.azure-pipelines/scripts/ut/compare_coverage.sh @@ -235,4 +235,3 @@ function main { } main - diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml index 03aa62877cb..629612275f2 100644 --- a/.azure-pipelines/template/docker-template.yml +++ b/.azure-pipelines/template/docker-template.yml @@ -15,7 +15,7 @@ parameters: type: string - name: repo type: string - default: 'https://github.com/VincyZhang/neural-compressor' + default: 'https://github.com/intel/neural-compressor' steps: - ${{ if eq(parameters.dockerConfigName, 'commonDockerConfig') }}: diff --git a/.azure-pipelines/template/ut-basic-template.yml b/.azure-pipelines/template/ut-basic-template.yml index 4627e5d1ad3..ea1a00f73e6 100644 --- a/.azure-pipelines/template/ut-basic-template.yml +++ b/.azure-pipelines/template/ut-basic-template.yml @@ -4,7 +4,7 @@ parameters: default: 'commonDockerConfig' - name: repo type: string - default: 'https://github.com/VincyZhang/neural-compressor' + default: 'https://github.com/intel/neural-compressor' - name: utScriptFileName type: string - name: uploadPath diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index 2e05b4113a1..c8bb04b7716 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -18,7 +18,7 @@ variables: UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir ARTIFACT_NAME: 'UT_coverage_report' - REPO: 'https://github.com/VincyZhang/neural-compressor' + REPO: 'https://github.com/intel/neural-compressor' stages: diff --git a/docker/README.md b/docker/README.md index af321afdc91..dd8b285c1ab 100644 --- a/docker/README.md +++ b/docker/README.md @@ -20,7 +20,7 @@ then please modify the tag so that the tagging requirement is met. For example r ```console $ PYTHON=python3.8 -$ INC_BRANCH=v1.13 +$ INC_BRANCH=v1.12 $ IMAGE_NAME=neural-compressor $ IMAGE_TAG=${INC_BRANCH}-devel $ docker build --build-arg PYTHON=${PYTHON} --build-arg INC_BRANCH=${INC_BRANCH} -f Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} . From de76b78b5e4ea366778c21cbb91f010931fa326f Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 24 Oct 2022 15:38:19 +0800 Subject: [PATCH 283/296] Baseline --- .azure-pipelines/model-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index dc30caff704..43790c10a62 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -302,7 +302,7 @@ stages: patterns: '**.log' path: $(OUT_SCRIPT_PATH)/last_generated project: $(System.TeamProject) - pipeline: $(System.DefinitionId) + pipeline: 'Baseline' runVersion: 'specific' runId: $(model_runID) retryDownloadCount: 3 From 5215b18ef5d303a30a102dee917f9e58e509f03e Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 24 Oct 2022 18:10:02 +0800 Subject: [PATCH 284/296] fix benchmark config --- .../scripts/models/collect_log_model.py | 21 ++++++++++++++----- .../scripts/models/run_benchmark_common.sh | 6 +++--- .../models/run_model_trigger_common.sh | 4 ++-- .../models/run_pytorch_models_trigger.sh | 4 ++-- .../scripts/models/update_yaml_config.py | 9 ++------ 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py index 7e2a5464a0b..e8aa5a91232 100644 --- a/.azure-pipelines/scripts/models/collect_log_model.py +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -43,19 +43,22 @@ def main(): # get model benchmark results for precision in ['int8', 'fp32']: throughput = 0.0 + bs = 1 for root, dirs, files in os.walk(args.logs_dir): for name in files: file_name = os.path.join(root, name) print(file_name) if 'performance-'+precision in name: for line in open(file_name, "r"): - result = parse_perf_line(line) - if result: - throughput += result + results = parse_perf_line(line) + if results["throughput"]: + throughput += results["throughput"] + if results["batch_size"]: + bs = results["batch_size"] # set model status failed if throughput==0.0: os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') - results.append('{};{};{};{};{};{};Inference;Performance;1;{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, precision.upper(), args.model, throughput, URL)) + results.append('{};{};{};{};{};{};Inference;Performance;{};{};{}\n'.format(OS, PLATFORM, args.framework, args.fwk_ver, precision.upper(), args.model, bs, throughput, URL)) # write model logs f = open(args.output_dir+'/'+args.framework+'_'+args.model+'_summary.log', "a") f.writelines("OS;Platform;Framework;Version;Precision;Model;Mode;Type;BS;Value;Url\n") @@ -107,9 +110,17 @@ def parse_tuning_line(line, tmp): def parse_perf_line(line) -> float: + perf_data = {} + throughput = re.search(r"Throughput:\s+(\d+(\.\d+)?)", line) if throughput and throughput.group(1): - return float(throughput.group(1)) + perf_data.update({"throughput": float(throughput.group(1))}) + + batch_size = re.search(r"Batch size = ([0-9]+)", line) + if batch_size and batch_size.group(1): + perf_data.update({"batch_size": int(batch_size.group(1))}) + + return perf_data if __name__ == '__main__': diff --git a/.azure-pipelines/scripts/models/run_benchmark_common.sh b/.azure-pipelines/scripts/models/run_benchmark_common.sh index 8f4be38103a..b342ac48992 100644 --- a/.azure-pipelines/scripts/models/run_benchmark_common.sh +++ b/.azure-pipelines/scripts/models/run_benchmark_common.sh @@ -82,9 +82,9 @@ function multiInstance() { cmd="${benchmark_cmd} --input_model=${input_model}" if [ "${new_benchmark}" == "true" ]; then - echo "run performance in precision ${precision}" + echo "run with internal benchmark..." eval ${cmd} 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log else -# multiInstance - multiInstance 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log + echo "run with external multiInstance benchmark..." + multiInstance fi diff --git a/.azure-pipelines/scripts/models/run_model_trigger_common.sh b/.azure-pipelines/scripts/models/run_model_trigger_common.sh index 87133d70717..fb4c9c236ad 100644 --- a/.azure-pipelines/scripts/models/run_model_trigger_common.sh +++ b/.azure-pipelines/scripts/models/run_model_trigger_common.sh @@ -110,12 +110,12 @@ else fi -# ======== update yaml config ======== +echo "======== update yaml config ========" echo -e "\nPrint origin yaml..." cat ${yaml} python ${SCRIPTS_PATH}/update_yaml_config.py --yaml=${yaml} --framework=${framework} \ --dataset_location=${dataset_location} --batch_size=${batch_size} --strategy=${strategy} \ ---new_benchmark=${new_benchmark} --multi_instance='false' +--new_benchmark=${new_benchmark} --multi_instance='true' echo -e "\nPrint updated yaml... " cat ${yaml} diff --git a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh index 795dd192ae7..6635aff497a 100644 --- a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh @@ -29,7 +29,7 @@ if [ "${model}" == "resnet18" ]; then input_model="" yaml="conf.yaml" strategy="bayesian" - batch_size=100 + batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=200" @@ -39,7 +39,7 @@ elif [ "${model}" == "resnet18_fx" ]; then input_model="" yaml="conf.yaml" strategy="basic" - batch_size=100 + batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=200" diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py index 1f37d8ba43f..97c28b1708b 100644 --- a/.azure-pipelines/scripts/models/update_yaml_config.py +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -271,19 +271,14 @@ def update_yaml_config_benchmark_acc(yaml_path: str, batch_size = None): def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_instance = None): # Get cpu information for multi-instance total_cores = psutil.cpu_count(logical=False) - # total_sockets = get_number_of_sockets() total_sockets = 1 ncores_per_socket = total_cores / total_sockets - - num_sockets = 1 # Use only one socket - num_benchmark_cores = ncores_per_socket * num_sockets - ncores_per_instance = ncores_per_socket iters = 100 if multi_instance=='true': ncores_per_instance = 4 - iters = 500 + iters = 200 with open(yaml_path) as f: yaml_config = yaml.round_trip_load(f, preserve_quotes=True) @@ -301,7 +296,7 @@ def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_i else: configs.update({ 'cores_per_instance': int(ncores_per_instance), - 'num_of_instance': int(num_benchmark_cores // ncores_per_instance) + 'num_of_instance': int(ncores_per_socket // ncores_per_instance) }) for attr in ['intra_num_of_threads', 'inter_num_of_threads', 'kmp_blocktime']: if configs.get(attr): From 56123b0b1314ccc457253ca931fa121a51f77354 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 24 Oct 2022 20:46:26 +0800 Subject: [PATCH 285/296] replace onnx model --- .azure-pipelines/model-test.yml | 10 +++++----- .../scripts/models/run_onnxrt_models_trigger.sh | 14 ++++++++++++-- .azure-pipelines/template/docker-template.yml | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index 43790c10a62..5f6fd909b87 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -228,17 +228,17 @@ stages: fi displayName: "Check Test Status" - - job: bert_base_MRPC_static - displayName: bert_base_MRPC_static + - job: distilbert_base_MRPC_qdq + displayName: distilbert_base_MRPC_qdq steps: - template: template/model-template.yml parameters: - modelName: 'bert_base_MRPC_static' + modelName: 'distilbert_base_MRPC_qdq' framework: 'onnxrt' - script: | - if [ $(onnxrt_bert_base_MRPC_static_failed) == 'true' ]; then - echo "[Failed] Model bert_base_MRPC_static failed, please check artifacts and logs." + if [ $(onnxrt_distilbert_base_MRPC_qdq_failed) == 'true' ]; then + echo "[Failed] Model distilbert_base_MRPC_qdq failed, please check artifacts and logs." exit 1 fi displayName: "Check Test Status" diff --git a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh index d445067807f..d8fbdec285e 100644 --- a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh @@ -38,7 +38,7 @@ elif [ "${model}" == "bert_base_MRPC_static" ]; then input_model="/tf_dataset2/models/onnx/bert_base_MRPC/bert.onnx" yaml="bert_static.yaml" strategy="basic" - batch_size=8 + batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" benchmark_cmd="bash run_benchmark.sh --config=${yaml}" @@ -48,7 +48,17 @@ elif [ "${model}" == "bert_base_MRPC_dynamic" ]; then input_model="/tf_dataset2/models/onnx/bert_base_MRPC/bert.onnx" yaml="bert_dynamic.yaml" strategy="basic" - batch_size=8 + batch_size=1 + new_benchmark=true + tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml}" +elif [ "${model}" == "distilbert_base_MRPC_qdq" ]; then + model_src_dir="language_translation/distilbert/quantization/ptq" + dataset_location="/tf_dataset/pytorch/glue_data/MRPC" + input_model="/tf_dataset2/models/onnx/distilbert_base_MRPC/distilbert-base-uncased.onnx" + yaml="distilbert_qdq.yaml" + strategy="basic" + batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" benchmark_cmd="bash run_benchmark.sh --config=${yaml}" diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml index 629612275f2..d3b25d3bcf8 100644 --- a/.azure-pipelines/template/docker-template.yml +++ b/.azure-pipelines/template/docker-template.yml @@ -59,7 +59,7 @@ steps: docker stop $(docker ps -aq) docker rm -vf $(docker ps -aq) || true env | sort - displayName: 'Clean Docker' + displayName: 'Clean docker container' - ${{ if ne(parameters.containerName, '') }}: - task: Bash@3 From 9f1d59b5ae8ad569013865b3471aa74c480acaf8 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 24 Oct 2022 20:57:49 +0800 Subject: [PATCH 286/296] bug fix --- .azure-pipelines/scripts/models/collect_log_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py index e8aa5a91232..039407c2cfe 100644 --- a/.azure-pipelines/scripts/models/collect_log_model.py +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -50,11 +50,11 @@ def main(): print(file_name) if 'performance-'+precision in name: for line in open(file_name, "r"): - results = parse_perf_line(line) - if results["throughput"]: - throughput += results["throughput"] - if results["batch_size"]: - bs = results["batch_size"] + result= parse_perf_line(line) + if result["throughput"]: + throughput += result["throughput"] + if result["batch_size"]: + bs = result["batch_size"] # set model status failed if throughput==0.0: os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') From 77f51b31c3698593d6d3039a014b99f4ea09326d Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 24 Oct 2022 22:16:27 +0800 Subject: [PATCH 287/296] bug fix --- .azure-pipelines/scripts/models/collect_log_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py index 039407c2cfe..37d305423bf 100644 --- a/.azure-pipelines/scripts/models/collect_log_model.py +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -51,10 +51,10 @@ def main(): if 'performance-'+precision in name: for line in open(file_name, "r"): result= parse_perf_line(line) - if result["throughput"]: - throughput += result["throughput"] - if result["batch_size"]: - bs = result["batch_size"] + if result.get("throughput"): + throughput += result.get("throughput") + if result.get("batch_size"): + bs = result.get("batch_size") # set model status failed if throughput==0.0: os.system('echo "##vso[task.setvariable variable='+args.framework+'_'+args.model+'_failed]true"') From c5043789e7ea69ca2d7bf583b1fe62dbb69ba033 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Mon, 24 Oct 2022 22:33:57 +0800 Subject: [PATCH 288/296] clean some print --- .azure-pipelines/model-test.yml | 1 - .azure-pipelines/ut-basic.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index 5f6fd909b87..ede0ae69458 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -289,7 +289,6 @@ stages: inputs: versionSpec: '3.8.0' - script: | - echo "------ Collecting logs ------" cd ${OUT_SCRIPT_PATH} mkdir generated mkdir last_generated diff --git a/.azure-pipelines/ut-basic.yml b/.azure-pipelines/ut-basic.yml index c8bb04b7716..9d49fab17eb 100644 --- a/.azure-pipelines/ut-basic.yml +++ b/.azure-pipelines/ut-basic.yml @@ -150,7 +150,6 @@ stages: path: $(DOWNLOAD_PATH) - script: | - echo "------ Collecting logs ------" echo "--- create container ---" docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash echo "--- docker ps ---" From 73587c4d7c883c3074aebf99394ec7b6db808820 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Tue, 25 Oct 2022 09:06:23 +0800 Subject: [PATCH 289/296] fix pytorch int8 benchmark --- .azure-pipelines/scripts/models/run_model_trigger_common.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/scripts/models/run_model_trigger_common.sh b/.azure-pipelines/scripts/models/run_model_trigger_common.sh index fb4c9c236ad..4306920e093 100644 --- a/.azure-pipelines/scripts/models/run_model_trigger_common.sh +++ b/.azure-pipelines/scripts/models/run_model_trigger_common.sh @@ -137,8 +137,11 @@ if [[ "${framework}" == "onnxrt" ]]; then model_name="${log_dir}/${model}/${framework}-${model}-tune.onnx" elif [[ "${framework}" == "mxnet" ]]; then model_name="${log_dir}/${model}" -else +elif [[ "${framework}" == "tensorflow" ]]; then model_name="${log_dir}/${model}/${framework}-${model}-tune.pb" +elif [[ "${framework}" == "pytorch" ]]; then + model_name=${input_model} + benchmark_cmd="${benchmark_cmd} --int8=true" fi /bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh --framework=${framework} --model=${model} \ --input_model="${model_name}" --benchmark_cmd="${benchmark_cmd}" \ From b946d2fea235323383353efe9c12402507c7696c Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 26 Oct 2022 00:34:18 +0800 Subject: [PATCH 290/296] update onnx version --- .azure-pipelines/model-test.yml | 6 +++--- .azure-pipelines/scripts/models/run_model_trigger_common.sh | 2 +- .../scripts/models/run_onnxrt_models_trigger.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index ede0ae69458..23f7ed163ee 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -13,11 +13,11 @@ parameters: - name: test_tf displayName: Run tensorflow models? type: boolean - default: true + default: false - name: test_pt displayName: Run pytorch models? type: boolean - default: true + default: false - name: test_onnx displayName: Run onnxrt models? type: boolean @@ -25,7 +25,7 @@ parameters: - name: test_mxnet displayName: Run mxnet models? type: boolean - default: true + default: false stages: - ${{ if eq(parameters.test_tf, true) }}: diff --git a/.azure-pipelines/scripts/models/run_model_trigger_common.sh b/.azure-pipelines/scripts/models/run_model_trigger_common.sh index 4306920e093..397ccade727 100644 --- a/.azure-pipelines/scripts/models/run_model_trigger_common.sh +++ b/.azure-pipelines/scripts/models/run_model_trigger_common.sh @@ -70,7 +70,7 @@ elif [[ "${framework}" == "pytorch" ]]; then pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html elif [[ "${framework}" == "onnxrt" ]]; then - pip install onnx==${fwk_ver} + pip install onnx==1.11.0 pip install onnxruntime==${fwk_ver} elif [[ "${framework}" == "mxnet" ]]; then if [[ "${fwk_ver}" == "1.7.0" ]]; then diff --git a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh index d8fbdec285e..d71081f61b3 100644 --- a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh @@ -18,7 +18,7 @@ do done FRAMEWORK="onnxrt" -FRAMEWORK_VERSION="1.11.0" +FRAMEWORK_VERSION="1.12.1" # ======== set up config for onnxrt models ======== From b26e8ffe9219b59bb426f04bbca2d6e12be79ee2 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 26 Oct 2022 00:41:19 +0800 Subject: [PATCH 291/296] corner issue, need to fix --- .azure-pipelines/model-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index 23f7ed163ee..ede0ae69458 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -13,11 +13,11 @@ parameters: - name: test_tf displayName: Run tensorflow models? type: boolean - default: false + default: true - name: test_pt displayName: Run pytorch models? type: boolean - default: false + default: true - name: test_onnx displayName: Run onnxrt models? type: boolean @@ -25,7 +25,7 @@ parameters: - name: test_mxnet displayName: Run mxnet models? type: boolean - default: false + default: true stages: - ${{ if eq(parameters.test_tf, true) }}: From 24e6b602841720e74f5eb71ca972c07a318103a2 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 26 Oct 2022 00:54:32 +0800 Subject: [PATCH 292/296] disable onnx bert_base_MRPC_dynamic && expend benchmark iteration --- .azure-pipelines/model-test.yml | 28 +++++++++---------- .../models/run_mxnet_models_trigger.sh | 2 +- .../models/run_pytorch_models_trigger.sh | 4 +-- .../models/run_tensorflow_models_trigger.sh | 14 +++++----- .../scripts/models/update_yaml_config.py | 2 +- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index ede0ae69458..c20156dcd8a 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -228,20 +228,20 @@ stages: fi displayName: "Check Test Status" - - job: distilbert_base_MRPC_qdq - displayName: distilbert_base_MRPC_qdq - steps: - - template: template/model-template.yml - parameters: - modelName: 'distilbert_base_MRPC_qdq' - framework: 'onnxrt' - - - script: | - if [ $(onnxrt_distilbert_base_MRPC_qdq_failed) == 'true' ]; then - echo "[Failed] Model distilbert_base_MRPC_qdq failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" +# - job: distilbert_base_MRPC_qdq +# displayName: distilbert_base_MRPC_qdq +# steps: +# - template: template/model-template.yml +# parameters: +# modelName: 'distilbert_base_MRPC_qdq' +# framework: 'onnxrt' +# +# - script: | +# if [ $(onnxrt_distilbert_base_MRPC_qdq_failed) == 'true' ]; then +# echo "[Failed] Model distilbert_base_MRPC_qdq failed, please check artifacts and logs." +# exit 1 +# fi +# displayName: "Check Test Status" - job: bert_base_MRPC_dynamic displayName: bert_base_MRPC_dynamic diff --git a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh index e082f5189d9..455ee809c61 100644 --- a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh @@ -28,7 +28,7 @@ if [ "${model}" == "resnet50v1" ]; then input_model="/tf_dataset/mxnet/resnet50_v1" yaml="cnn.yaml" strategy="mse" - batch_size=32 + batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=resnet50_v1 --dataset_location=${dataset_location} --input_model=${input_model}" benchmark_cmd="bash run_benchmark.sh --topology=resnet50_v1 --dataset_location=${dataset_location} --batch_size=1 --iters=500 --mode=benchmark" diff --git a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh index 6635aff497a..9aef437666d 100644 --- a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh @@ -32,7 +32,7 @@ if [ "${model}" == "resnet18" ]; then batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" - benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=200" + benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=500" elif [ "${model}" == "resnet18_fx" ]; then model_src_dir="image_recognition/torchvision_models/quantization/ptq/cpu/fx/" dataset_location="/tf_dataset2/datasets/mini-imageraw" @@ -42,7 +42,7 @@ elif [ "${model}" == "resnet18_fx" ]; then batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=resnet18 --dataset_location=${dataset_location} --input_model=${input_model}" - benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=200" + benchmark_cmd="bash run_benchmark.sh --topology=resnet18 --dataset_location=${dataset_location} --mode=benchmark --batch_size=${batch_size} --iters=500" fi diff --git a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh index 581477f4667..1b3a6c6815d 100644 --- a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh @@ -27,7 +27,7 @@ if [ "${model}" == "resnet50v1.5" ]; then input_model="/tf_dataset/pre-trained-models/resnet50v1_5/fp32/resnet50_v1.pb" yaml="resnet50_v1_5.yaml" strategy="basic" - batch_size=100 + batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" @@ -57,7 +57,7 @@ elif [ "${model}" == "inception_v1" ]; then input_model="/tf_dataset/pre-train-model-slim/pbfile/frozen_pb/frozen_inception_v1.pb" yaml="inception_v1.yaml" strategy="basic" - batch_size=100 + batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --config=${yaml} --input_model=${input_model}" benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" @@ -67,20 +67,20 @@ elif [ "${model}" == "darknet19" ]; then input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/PublicInHouse/classification/darknet19/darknet19.pb" yaml="config.yaml" strategy="basic" - batch_size=100 + batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" - benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=200" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500" elif [ "${model}" == "densenet-121" ]; then model_src_dir="oob_models/quantization/ptq" dataset_location="" input_model="/tf_dataset/tensorflow/tf_oob_models/ov/all_tf_models/classification/densenet/121/tf/densenet-121.pb" yaml="config.yaml" strategy="basic" - batch_size=100 + batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" - benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=200" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500" elif [ "${model}" == "resnet-101" ]; then model_src_dir="oob_models/quantization/ptq" dataset_location="" @@ -90,7 +90,7 @@ elif [ "${model}" == "resnet-101" ]; then batch_size=1 new_benchmark=false tuning_cmd="bash run_tuning.sh --topology=${model} --dataset_location= --input_model=${input_model}" - benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=200" + benchmark_cmd="bash run_benchmark.sh --topology=${model} --dataset_location= --mode=benchmark --batch_size=1 --iters=500" elif [ "${model}" == "resnet50_fashion" ]; then model_src_dir="image_recognition/keras_models/resnet50_fashion/quantization/ptq" dataset_location="/tf_dataset2/datasets/mnist/FashionMNIST_small" diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py index 97c28b1708b..e1592e18ecf 100644 --- a/.azure-pipelines/scripts/models/update_yaml_config.py +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -278,7 +278,7 @@ def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_i if multi_instance=='true': ncores_per_instance = 4 - iters = 200 + iters = 500 with open(yaml_path) as f: yaml_config = yaml.round_trip_load(f, preserve_quotes=True) From eeaa900ada72bc37be0d83f68acb0f59da29b9bf Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Wed, 26 Oct 2022 16:31:44 +0800 Subject: [PATCH 293/296] add link --- .azure-pipelines/model-test.yml | 2 +- .../scripts/models/generate_report.sh | 42 ++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index c20156dcd8a..1f23a460563 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -309,7 +309,7 @@ stages: - script: | echo "------ Generating final report.html ------" cd ${OUT_SCRIPT_PATH} - /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated + /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated --ghprbPullId $(System.PullRequest.PullRequestNumber) --MR_source_branch $(System.PullRequest.SourceBranch) --MR_source_repo $(System.PullRequest.SourceRepositoryURI) --MR_target_branch $(System.PullRequest.TargetBranch) --repo_url $(Build.Repository.Uri) --source_commit_id $(Build.SourceVersion) --build_id $(Build.BuildId) displayName: 'Generate report' - task: PublishPipelineArtifact@1 inputs: diff --git a/.azure-pipelines/scripts/models/generate_report.sh b/.azure-pipelines/scripts/models/generate_report.sh index c5cbb6080f5..51b0e3423c8 100644 --- a/.azure-pipelines/scripts/models/generate_report.sh +++ b/.azure-pipelines/scripts/models/generate_report.sh @@ -39,6 +39,34 @@ while [[ $# -gt 0 ]];do last_logt_dir=${2} shift 2 ;; + --ghprbPullId) + ghprbPullId=${2} + shift 2 + ;; + --MR_source_branch) + MR_source_branch=${2} + shift 2 + ;; + --MR_source_repo) + MR_source_repo=${2} + shift 2 + ;; + --MR_target_branch) + MR_target_branch=${2} + shift 2 + ;; + --repo_url) + repo_url=${2} + shift 2 + ;; + --source_commit_id) + source_commit_id=${2} + shift 2 + ;; + --build_id) + build_id=${2} + shift 2 + ;; *) shift ;; @@ -58,7 +86,11 @@ summaryLogLast="${last_logt_dir}/summary.log" tuneLogLast="${last_logt_dir}/tuning_info.log" echo "summaryLogLast: ${summaryLogLast}" echo "tuneLogLast: ${tuneLogLast}" - +echo "MR_source_branch: ${MR_source_branch}" +echo "MR_source_repo: ${MR_source_repo}" +echo "MR_target_branch: ${MR_target_branch}" +echo "repo_url: ${repo_url}" +echo "commit_id: ${source_commit_id}" function main { @@ -396,9 +428,9 @@ if [ "${qtools_branch}" == "" ]; then commit_id=$(echo ${ghprbActualCommit} |awk '{print substr($1,1,7)}') - MR_TITLE="[ PR-${ghprbPullId} ]" + MR_TITLE="[ PR-${ghprbPullId} ]" Test_Info_Title="Source Branch Target Branch Commit " - Test_Info="${MR_source_branch} ${MR_target_branch} ${commit_id}" + Test_Info="${MR_source_branch} ${MR_target_branch} ${source_commit_id:0:6}" else Test_Info_Title="Test Branch Commit ID " Test_Info="${qtools_branch} ${qtools_commit} " @@ -409,7 +441,7 @@ cat >> ${output_dir}/report.html << eof

Neural Compressor Tuning Tests ${MR_TITLE} - [ Job-${BUILD_NUMBER} ]

+ [ Job-${build_id} ]

Test Status: ${Jenkins_job_status}

Summary

@@ -418,7 +450,7 @@ cat >> ${output_dir}/report.html << eof ${Test_Info_Title} - + ${Test_Info}
neural-compressorneural-compressor
From 8243afafa75a0caeb4562a9482d522fb999259e3 Mon Sep 17 00:00:00 2001 From: chensuyue Date: Wed, 26 Oct 2022 20:47:45 +0800 Subject: [PATCH 294/296] remove some debug print --- .azure-pipelines/scripts/models/update_yaml_config.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py index e1592e18ecf..9d92a457f6d 100644 --- a/.azure-pipelines/scripts/models/update_yaml_config.py +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -62,7 +62,6 @@ def update_yaml_dataset(yaml, framework, dataset_location): if re.search(key_patterns["pattern"], line): print(f"Replacing {key} key.") line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) - # print(line) config.write(line) if framework == "pytorch": @@ -94,7 +93,6 @@ def update_yaml_dataset(yaml, framework, dataset_location): if re.search(key_patterns["pattern"], line): print(f"Replacing {key} key.") line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) - # print(line) config.write(line) @@ -236,8 +234,6 @@ def update_yaml_config_tuning(yaml_file, strategy = None, mode = None, batch_siz yaml_content = yaml.round_trip_dump(yaml_config) - # print(yaml_content) - with open(yaml_file, 'w') as output_file: output_file.write(yaml_content) @@ -262,8 +258,6 @@ def update_yaml_config_benchmark_acc(yaml_path: str, batch_size = None): yaml_content = yaml.round_trip_dump(yaml_config) - # print(yaml_content) - with open(yaml_path, 'w') as output_file: output_file.write(yaml_content) @@ -309,8 +303,6 @@ def update_yaml_config_benchmark_perf(yaml_path: str, batch_size = None, multi_i yaml_content = yaml.round_trip_dump(yaml_config) - # print(yaml_content) - with open(yaml_path, 'w') as output_file: output_file.write(yaml_content) From 3cab0a0ec5283bd729b609280fd3c80c8e55562c Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Sun, 30 Oct 2022 20:14:05 +0800 Subject: [PATCH 295/296] Optimize model template (#123) --- .azure-pipelines/model-test.yml | 353 ++++++------------- .azure-pipelines/template/model-template.yml | 27 +- 2 files changed, 120 insertions(+), 260 deletions(-) diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index 1f23a460563..f604b176e13 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -10,284 +10,137 @@ variables: SCRIPT_PATH: /neural-compressor/.azure-pipelines/scripts parameters: -- name: test_tf - displayName: Run tensorflow models? - type: boolean - default: true -- name: test_pt - displayName: Run pytorch models? - type: boolean - default: true -- name: test_onnx - displayName: Run onnxrt models? - type: boolean - default: true -- name: test_mxnet - displayName: Run mxnet models? - type: boolean - default: true + - name: TensorFlow_Model + displayName: Run TensorFlow models? + type: boolean + default: true + - name: PyTorch_Model + displayName: Run PyTorch models? + type: boolean + default: true + - name: ONNX_Model + displayName: Run ONNX models? + type: boolean + default: true + - name: MXNet_Model + displayName: Run MXNet models? + type: boolean + default: true + + - name: TensorFlowModelList + type: object + default: + - resnet50v1.5 + - ssd_resnet50_v1 + - ssd_mobilenet_v1_ckpt + - inception_v1 + - resnet50_fashion + - darknet19 + - densenet-121 + - resnet-101 + - name: PyTorchModelList + type: object + default: + - resnet18 + - resnet18_fx + - name: ONNXModelList + type: object + default: + - resnet50-v1-12 + - bert_base_MRPC_dynamic + - name: MXNetModelList + type: object + default: + - resnet50v1 + stages: -- ${{ if eq(parameters.test_tf, true) }}: - - stage: TensorflowModels - displayName: Tensorflow Model Test + - stage: TensorFlowModels + displayName: Run TensorFlow Model dependsOn: [] + condition: eq('${{ parameters.TensorFlow_Model }}', 'true') jobs: - - job: resnet50v1_5 - displayName: resnet50v1.5 - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet50v1.5' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_resnet50v1.5_failed) == 'true' ]; then - echo "[Failed] Model resnet50v1.5 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: ssd_resnet50_v1 - displayName: ssd_resnet50_v1 - steps: - - template: template/model-template.yml - parameters: - modelName: 'ssd_resnet50_v1' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_ssd_resnet50_v1_failed) == 'true' ]; then - echo "[Failed] Model ssd_resnet50_v1 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: ssd_mobilenet_v1_ckpt - displayName: ssd_mobilenet_v1_ckpt - steps: - - template: template/model-template.yml - parameters: - modelName: 'ssd_mobilenet_v1_ckpt' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_ssd_mobilenet_v1_ckpt_failed) == 'true' ]; then - echo "[Failed] Model ssd_mobilenet_v1_ckpt failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: inception_v1 - displayName: inception_v1 - steps: - - template: template/model-template.yml - parameters: - modelName: 'inception_v1' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_inception_v1_failed) == 'true' ]; then - echo "[Failed] Model inception_v1 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: resnet50_fashion - displayName: resnet50_fashion - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet50_fashion' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_resnet50_fashion_failed) == 'true' ]; then - echo "[Failed] Model resnet50_fashion failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: darknet19 - displayName: darknet19 - steps: - - template: template/model-template.yml - parameters: - modelName: 'darknet19' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_darknet19_failed) == 'true' ]; then - echo "[Failed] Model darknet19 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: densenet121 - displayName: densenet-121 - steps: - - template: template/model-template.yml - parameters: - modelName: 'densenet-121' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_densenet-121_failed) == 'true' ]; then - echo "[Failed] Model densenet-121 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" + - ${{ each model in parameters.TensorFlowModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "tensorflow" - - job: resnet101 - displayName: resnet-101 - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet-101' - framework: 'tensorflow' - - - script: | - if [ $(tensorflow_resnet-101_failed) == 'true' ]; then - echo "[Failed] Model resnet-101 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - -- ${{ if eq(parameters.test_pt, true) }}: - stage: PyTorchModels - displayName: PyTorch Model Test + displayName: Run PyTorch Model dependsOn: [] + condition: eq('${{ parameters.PyTorch_Model }}', 'true') jobs: - - job: resnet18 - displayName: resnet18 - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet18' - framework: 'pytorch' - - - script: | - if [ $(pytorch_resnet18_failed) == 'true' ]; then - echo "[Failed] Model resnet18 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - - - job: resnet18_fx - displayName: resnet18_fx - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet18_fx' - framework: 'pytorch' - - - script: | - if [ $(pytorch_resnet18_fx_failed) == 'true' ]; then - echo "[Failed] Model resnet18_fx failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" + - ${{ each model in parameters.PyTorchModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "pytorch" -- ${{ if eq(parameters.test_mxnet, true) }}: - - stage: MXNetModels - displayName: Mxnet Model Test + - stage: ONNXModels + displayName: Run ONNX Model dependsOn: [] + condition: eq('${{ parameters.ONNX_Model }}', 'true') jobs: - - job: resnet50v1 - displayName: resnet50v1 - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet50v1' - framework: 'mxnet' - - - script: | - if [ $(mxnet_resnet50v1_failed) == 'true' ]; then - echo "[Failed] Model resnet50v1 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" + - ${{ each model in parameters.ONNXModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "onnxrt" -- ${{ if eq(parameters.test_onnx, true) }}: - - stage: ONNXModels - displayName: ONNXrt Model Test + - stage: MXNetModels + displayName: Run MXNet Model dependsOn: [] + condition: eq('${{ parameters.MXNet_Model }}', 'true') jobs: - - job: resnet50_v1_12 - displayName: resnet50-v1-12 - steps: - - template: template/model-template.yml - parameters: - modelName: 'resnet50-v1-12' - framework: 'onnxrt' - - - script: | - if [ $(onnxrt_resnet50-v1-12_failed) == 'true' ]; then - echo "[Failed] Model resnet50-v1-12 failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - -# - job: distilbert_base_MRPC_qdq -# displayName: distilbert_base_MRPC_qdq -# steps: -# - template: template/model-template.yml -# parameters: -# modelName: 'distilbert_base_MRPC_qdq' -# framework: 'onnxrt' -# -# - script: | -# if [ $(onnxrt_distilbert_base_MRPC_qdq_failed) == 'true' ]; then -# echo "[Failed] Model distilbert_base_MRPC_qdq failed, please check artifacts and logs." -# exit 1 -# fi -# displayName: "Check Test Status" + - ${{ each model in parameters.MXNetModelList }}: + - job: + displayName: ${{ model }} + steps: + - template: template/model-template.yml + parameters: + modelName: ${{ model }} + framework: "mxnet" - - job: bert_base_MRPC_dynamic - displayName: bert_base_MRPC_dynamic - steps: - - template: template/model-template.yml - parameters: - modelName: 'bert_base_MRPC_dynamic' - framework: 'onnxrt' - - - script: | - if [ $(onnxrt_bert_base_MRPC_dynamic_failed) == 'true' ]; then - echo "[Failed] Model bert_base_MRPC_dynamic failed, please check artifacts and logs." - exit 1 - fi - displayName: "Check Test Status" - stage: GenerateLogs displayName: Generate Report - dependsOn: [ TensorflowModels, PyTorchModels, MXNetModels, ONNXModels ] - condition: succeededOrFailed() + dependsOn: [TensorFlowModels, PyTorchModels, MXNetModels, ONNXModels] + condition: always() jobs: - job: GenerateReport - displayName: generate report steps: - script: | echo ${BUILD_SOURCESDIRECTORY} sudo rm -fr ${BUILD_SOURCESDIRECTORY} || true echo y | docker system prune - displayName: 'Clean workspace' + displayName: "Clean workspace" - checkout: self clean: true - displayName: 'Checkout out Repo' + displayName: "Checkout out Repo" - task: DownloadPipelineArtifact@2 inputs: artifact: - patterns: '**/*_summary.log' + patterns: "**/*_summary.log" path: $(OUT_SCRIPT_PATH) - task: DownloadPipelineArtifact@2 inputs: artifact: - patterns: '**/*_tuning_info.log' + patterns: "**/*_tuning_info.log" path: $(OUT_SCRIPT_PATH) - task: UsePythonVersion@0 - displayName: 'Use Python 3.8.0' + displayName: "Use Python 3." inputs: - versionSpec: '3.8.0' + versionSpec: "3" - script: | cd ${OUT_SCRIPT_PATH} mkdir generated @@ -296,30 +149,30 @@ stages: displayName: "Collect all logs" - task: DownloadPipelineArtifact@2 inputs: - source: 'specific' - artifact: 'FinalReport' - patterns: '**.log' + source: "specific" + artifact: "FinalReport" + patterns: "**.log" path: $(OUT_SCRIPT_PATH)/last_generated project: $(System.TeamProject) - pipeline: 'Baseline' - runVersion: 'specific' + pipeline: "Baseline" + runVersion: "specific" runId: $(model_runID) retryDownloadCount: 3 - displayName: 'Download last logs' + displayName: "Download last logs" - script: | echo "------ Generating final report.html ------" cd ${OUT_SCRIPT_PATH} /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated --ghprbPullId $(System.PullRequest.PullRequestNumber) --MR_source_branch $(System.PullRequest.SourceBranch) --MR_source_repo $(System.PullRequest.SourceRepositoryURI) --MR_target_branch $(System.PullRequest.TargetBranch) --repo_url $(Build.Repository.Uri) --source_commit_id $(Build.SourceVersion) --build_id $(Build.BuildId) - displayName: 'Generate report' + displayName: "Generate report" - task: PublishPipelineArtifact@1 inputs: targetPath: $(OUT_SCRIPT_PATH)/generated artifact: FinalReport - publishLocation: 'pipeline' + publishLocation: "pipeline" displayName: "Publish report" - script: | if [ $(is_perf_reg) == 'true' ]; then echo "[Performance Regression] Some model performance regression occurred, please check artifacts and reports." exit 1 fi - displayName: "Specify performance regression" \ No newline at end of file + displayName: "Specify performance regression" diff --git a/.azure-pipelines/template/model-template.yml b/.azure-pipelines/template/model-template.yml index fdab45aaaea..1bc9dd4a7bf 100644 --- a/.azure-pipelines/template/model-template.yml +++ b/.azure-pipelines/template/model-template.yml @@ -1,14 +1,14 @@ parameters: -- name: modelName - type: string - default: 'resnet50v1.5' -- name: framework - type: string - default: 'tensorflow' + - name: modelName + type: string + default: "resnet50v1.5" + - name: framework + type: string + default: "tensorflow" -- name: modelContainerName - type: string - default: 'model' + - name: modelContainerName + type: string + default: "model" steps: - template: docker-template.yml @@ -28,4 +28,11 @@ steps: inputs: targetPath: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}/ artifact: ${{ parameters.framework }}_${{ parameters.modelName }} - publishLocation: 'pipeline' \ No newline at end of file + publishLocation: "pipeline" + + - script: | + if [ ${{ parameters.framework }}_${{ parameters.modelName }}_failed == 'true' ]; then + echo "[Failed] Model ${{ parameters.modelName }} failed, please check artifacts and logs." + exit 1 + fi + displayName: "Check Test Status" From c62f6704dff8af6b9dc60c347aa499e7eefe245c Mon Sep 17 00:00:00 2001 From: "chen, suyue" Date: Tue, 15 Nov 2022 11:42:11 +0800 Subject: [PATCH 296/296] Perf stable check (#129) --- .azure-pipelines/model-test.yml | 14 +- .azure-pipelines/scripts/install_nc.sh | 2 +- .azure-pipelines/scripts/install_nc_full.sh | 2 +- .../scripts/models/collect_log_all.py | 4 +- .../scripts/models/collect_log_model.py | 145 +++++++++++- .azure-pipelines/scripts/models/env_setup.sh | 124 ++++++++++ .../scripts/models/generate_report.sh | 215 ++++++++---------- .../scripts/models/run_benchmark_common.sh | 117 ++++++---- .../models/run_model_trigger_common.sh | 195 +++++++--------- .../models/run_mxnet_models_trigger.sh | 26 ++- .../models/run_onnxrt_models_trigger.sh | 34 +-- .../models/run_pytorch_models_trigger.sh | 28 ++- .../models/run_tensorflow_models_trigger.sh | 26 ++- .../scripts/models/run_tuning_common.sh | 18 +- .../scripts/models/update_yaml_config.py | 14 +- .azure-pipelines/template/model-template.yml | 50 +++- 16 files changed, 659 insertions(+), 355 deletions(-) create mode 100644 .azure-pipelines/scripts/models/env_setup.sh diff --git a/.azure-pipelines/model-test.yml b/.azure-pipelines/model-test.yml index f604b176e13..c63d03cc6a4 100644 --- a/.azure-pipelines/model-test.yml +++ b/.azure-pipelines/model-test.yml @@ -47,13 +47,12 @@ parameters: type: object default: - resnet50-v1-12 - - bert_base_MRPC_dynamic + # - bert_base_MRPC_dynamic - name: MXNetModelList type: object default: - resnet50v1 - stages: - stage: TensorFlowModels displayName: Run TensorFlow Model @@ -111,7 +110,6 @@ stages: modelName: ${{ model }} framework: "mxnet" - - stage: GenerateLogs displayName: Generate Report dependsOn: [TensorFlowModels, PyTorchModels, MXNetModels, ONNXModels] @@ -138,9 +136,9 @@ stages: patterns: "**/*_tuning_info.log" path: $(OUT_SCRIPT_PATH) - task: UsePythonVersion@0 - displayName: "Use Python 3." + displayName: "Use Python 3.8" inputs: - versionSpec: "3" + versionSpec: "3.8" - script: | cd ${OUT_SCRIPT_PATH} mkdir generated @@ -154,15 +152,15 @@ stages: patterns: "**.log" path: $(OUT_SCRIPT_PATH)/last_generated project: $(System.TeamProject) - pipeline: "Baseline" + pipeline: "Model-Test" runVersion: "specific" - runId: $(model_runID) + runId: $(refer_buildId) retryDownloadCount: 3 displayName: "Download last logs" - script: | echo "------ Generating final report.html ------" cd ${OUT_SCRIPT_PATH} - /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated --ghprbPullId $(System.PullRequest.PullRequestNumber) --MR_source_branch $(System.PullRequest.SourceBranch) --MR_source_repo $(System.PullRequest.SourceRepositoryURI) --MR_target_branch $(System.PullRequest.TargetBranch) --repo_url $(Build.Repository.Uri) --source_commit_id $(Build.SourceVersion) --build_id $(Build.BuildId) + /usr/bin/bash generate_report.sh --WORKSPACE generated --output_dir generated --last_logt_dir last_generated displayName: "Generate report" - task: PublishPipelineArtifact@1 inputs: diff --git a/.azure-pipelines/scripts/install_nc.sh b/.azure-pipelines/scripts/install_nc.sh index 38fd439404b..fa2daca5d19 100644 --- a/.azure-pipelines/scripts/install_nc.sh +++ b/.azure-pipelines/scripts/install_nc.sh @@ -4,4 +4,4 @@ cd /neural-compressor python -m pip install --no-cache-dir -r requirements.txt python setup.py sdist bdist_wheel pip install dist/neural_compressor*.whl -pip list \ No newline at end of file +pip list diff --git a/.azure-pipelines/scripts/install_nc_full.sh b/.azure-pipelines/scripts/install_nc_full.sh index b3f59626dfd..7513baeb254 100644 --- a/.azure-pipelines/scripts/install_nc_full.sh +++ b/.azure-pipelines/scripts/install_nc_full.sh @@ -4,4 +4,4 @@ cd /neural-compressor python -m pip install --no-cache-dir -r requirements.txt python setup.py --full sdist bdist_wheel pip install dist/neural_compressor*.whl -pip list \ No newline at end of file +pip list diff --git a/.azure-pipelines/scripts/models/collect_log_all.py b/.azure-pipelines/scripts/models/collect_log_all.py index 61fe9454c2e..fb9db0d6721 100644 --- a/.azure-pipelines/scripts/models/collect_log_all.py +++ b/.azure-pipelines/scripts/models/collect_log_all.py @@ -1,7 +1,5 @@ -import re -import os -import platform import argparse +import os parser = argparse.ArgumentParser(allow_abbrev=False) parser.add_argument("--logs_dir", type=str, default=".") diff --git a/.azure-pipelines/scripts/models/collect_log_model.py b/.azure-pipelines/scripts/models/collect_log_model.py index 37d305423bf..7fbfb55dfac 100644 --- a/.azure-pipelines/scripts/models/collect_log_model.py +++ b/.azure-pipelines/scripts/models/collect_log_model.py @@ -1,8 +1,6 @@ -import re -import os -import platform import argparse - +import os +import re parser = argparse.ArgumentParser(allow_abbrev=False) parser.add_argument("--framework", type=str, required=True) @@ -11,6 +9,7 @@ parser.add_argument("--logs_dir", type=str, default=".") parser.add_argument("--output_dir", type=str, default=".") parser.add_argument("--build_id", type=str, default="3117") +parser.add_argument("--stage", type=str, default="collect_log") args = parser.parse_args() print('===== collecting log model =======') print('build_id: '+args.build_id) @@ -18,13 +17,119 @@ PLATFORM='icx' URL ='https://dev.azure.com/lpot-inc/neural-compressor/_build/results?buildId='+args.build_id+'&view=artifacts&pathAsName=false&type=publishedArtifacts' -print(args) + +def get_model_tuning_dict_results(): + tuning_result_dict = {} + + if os.path.exists(tuning_log): + print('tuning log found') + tmp = {'fp32_acc': 0, 'int8_acc': 0, 'tuning_trials': 0} + with open(tuning_log, "r") as f: + for line in f: + parse_tuning_line(line, tmp) + print(tmp) + # set model status failed + if tmp['fp32_acc'] == 0 or tmp['int8_acc'] == 0: + os.system('echo "##vso[task.setvariable variable=' + args.framework + '_' + args.model + '_failed]true"') + + tuning_result_dict = { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Strategy": tmp['strategy'], + "Tune_time": tmp['tune_time'], + } + benchmark_accuracy_result_dict = { + 'int8': { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Mode": "Inference", + "Type": "Accuracy", + "BS": 1, + "Value": tmp['int8_acc'], + "Url": URL, + }, + 'fp32': { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Mode": "Inference", + "Type": "Accuracy", + "BS": 1, + "Value": tmp['fp32_acc'], + "Url": URL, + } + } + + return tuning_result_dict, benchmark_accuracy_result_dict + else: + return {}, {} + + +def get_model_benchmark_dict_results(): + benchmark_performance_result_dict = {"int8": {}, "fp32": {}} + for precision in ["int8", "fp32"]: + throughput = 0.0 + bs = 1 + for root, dirs, files in os.walk(args.logs_dir): + for name in files: + file_name = os.path.join(root, name) + print(file_name) + if "performance-" + precision in name: + for line in open(file_name, "r"): + result = parse_perf_line(line) + if result.get("throughput"): + throughput += result.get("throughput") + if result.get("batch_size"): + bs = result.get("batch_size") + + # set model status failed + if throughput == 0.0: + os.system('echo "##vso[task.setvariable variable=' + args.framework + '_' + args.model + '_failed]true"') + benchmark_performance_result_dict[precision] = { + "OS": OS, + "Platform": PLATFORM, + "Framework": args.framework, + "Version": args.fwk_ver, + "Model": args.model, + "Mode": "Inference", + "Type": "Performance", + "BS": 1, + "Value": throughput, + "Url": URL, + } + + return benchmark_performance_result_dict + + +def get_refer_data(): + refer_log = os.path.join(f"{args.logs_dir}_refer_log", f"{args.framework}_{args.model}_summary.log") + result = {} + if os.path.exists(refer_log): + with open(refer_log, "r") as f: + lines = f.readlines() + keys = lines[0].split(";") + values = [lines[i].split(";") for i in range(1, len(lines))] + for value in values: + precision = value[keys.index("Precision")] + Type = value[keys.index("Type")] + result[f"{precision}_{Type}"] = float(value[keys.index("Value")]) + return result + else: + print(f"refer log file: {refer_log} not found") + return 0 -def main(): +def collect_log(): results = [] tuning_infos = [] - tuning_log = os.path.join(args.logs_dir, f"{args.framework}-{args.model}-tune.log") print("tuning log dir is {}".format(tuning_log)) # get model tuning results if os.path.exists(tuning_log): @@ -123,5 +228,29 @@ def parse_perf_line(line) -> float: return perf_data +def check_status(precision, precision_upper, check_accuracy = False): + performance_result = get_model_benchmark_dict_results() + current_performance = performance_result.get(precision).get("Value") + refer_performance = refer.get(f"{precision_upper}_Performance") + print(f"current_performance_data = {current_performance}, refer_performance_data = {refer_performance}") + assert abs(current_performance - refer_performance) / refer_performance <= 0.05 + + if check_accuracy: + _, accuracy_result = get_model_tuning_dict_results() + current_accuracy = accuracy_result.get(precision).get("Value") + refer_accuracy = refer.get(f"{precision_upper}_Accuracy") + print(f"current_accuracy_data = {current_accuracy}, refer_accuarcy_data = {refer_accuracy}") + assert abs(current_accuracy - refer_accuracy) / refer_accuracy <= 0.05 + + if __name__ == '__main__': - main() + tuning_log = os.path.join(args.logs_dir, f"{args.framework}-{args.model}-tune.log") + refer = get_refer_data() + if args.stage == "collect_log": + collect_log() + elif args.stage == "int8_benchmark": + check_status("int8", "INT8") + elif args.stage == "fp32_benchmark": + check_status("fp32", "FP32") + else: + raise ValueError(f"{args.stage} does not exist") diff --git a/.azure-pipelines/scripts/models/env_setup.sh b/.azure-pipelines/scripts/models/env_setup.sh new file mode 100644 index 00000000000..7443e3e9d25 --- /dev/null +++ b/.azure-pipelines/scripts/models/env_setup.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh + +# get parameters +PATTERN='[-a-zA-Z0-9_]*=' + +for i in "$@"; do + case $i in + --yaml=*) + yaml=$(echo $i | sed "s/${PATTERN}//") + ;; + --framework=*) + framework=$(echo $i | sed "s/${PATTERN}//") + ;; + --fwk_ver=*) + fwk_ver=$(echo $i | sed "s/${PATTERN}//") + ;; + --torch_vision_ver=*) + torch_vision_ver=$(echo $i | sed "s/${PATTERN}//") + ;; + --model=*) + model=$(echo $i | sed "s/${PATTERN}//") + ;; + --model_src_dir=*) + model_src_dir=$(echo $i | sed "s/${PATTERN}//") + ;; + --dataset_location=*) + dataset_location=$(echo $i | sed "s/${PATTERN}//") + ;; + --batch_size=*) + batch_size=$(echo $i | sed "s/${PATTERN}//") + ;; + --strategy=*) + strategy=$(echo $i | sed "s/${PATTERN}//") + ;; + --new_benchmark=*) + new_benchmark=$(echo $i | sed "s/${PATTERN}//") + ;; + *) + echo "Parameter $i not recognized." + exit 1 + ;; + esac +done + +SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" +log_dir="/neural-compressor/.azure-pipelines/scripts/models" +WORK_SOURCE_DIR="/neural-compressor/examples/${framework}" +$BOLD_YELLOW && echo "processing ${framework}-${fwk_ver}-${model}" && $RESET + +$BOLD_YELLOW && echo "======= creat log_dir =========" && $RESET +if [ -d "${log_dir}/${model}" ]; then + $BOLD_GREEN && echo "${log_dir}/${model} already exists, don't need to mkdir." && $RESET +else + $BOLD_GREEN && echo "no log dir ${log_dir}/${model}, create." && $RESET + cd ${log_dir} + mkdir ${model} +fi + +$BOLD_YELLOW && echo "====== install requirements ======" && $RESET +/bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh + +cd ${WORK_SOURCE_DIR}/${model_src_dir} +pip install ruamel_yaml +pip install psutil +pip install protobuf==3.20.1 +if [[ "${framework}" == "tensorflow" ]]; then + pip install intel-tensorflow==${fwk_ver} +elif [[ "${framework}" == "pytorch" ]]; then + pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html + pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html +elif [[ "${framework}" == "onnxrt" ]]; then + pip install onnx==1.11.0 + pip install onnxruntime==${fwk_ver} +elif [[ "${framework}" == "mxnet" ]]; then + if [[ "${fwk_ver}" == "1.7.0" ]]; then + pip install mxnet==${fwk_ver}.post2 + elif [[ "${fwk_ver}" == "1.6.0" ]]; then + pip install mxnet-mkl==${mxnet_version} + else + pip install mxnet==${fwk_ver} + fi +fi + +if [ -f "requirements.txt" ]; then + sed -i '/neural-compressor/d' requirements.txt + if [ "${framework}" == "onnxrt" ]; then + sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt + fi + if [ "${framework}" == "tensorflow" ]; then + sed -i '/tensorflow==/d;/tensorflow$/d' requirements.txt + sed -i '/^intel-tensorflow/d' requirements.txt + fi + if [ "${framework}" == "mxnet" ]; then + sed -i '/mxnet==/d;/mxnet$/d;/mxnet-mkl==/d;/mxnet-mkl$/d' requirements.txt + fi + if [ "${framework}" == "pytorch" ]; then + sed -i '/torch==/d;/torch$/d;/torchvision==/d;/torchvision$/d' requirements.txt + fi + n=0 + until [ "$n" -ge 5 ]; do + python -m pip install -r requirements.txt && break + n=$((n + 1)) + sleep 5 + done + pip list +else + $BOLD_RED && echo "Not found requirements.txt file." && $RESET +fi + +$BOLD_YELLOW && echo "======== update yaml config ========" && $RESET +$BOLD_YELLOW && echo -e "\nPrint origin yaml..." && $RESET +cat ${yaml} +python ${SCRIPTS_PATH}/update_yaml_config.py \ + --yaml=${yaml} \ + --framework=${framework} \ + --dataset_location=${dataset_location} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --multi_instance='true' +$BOLD_YELLOW && echo -e "\nPrint updated yaml... " && $RESET +cat ${yaml} diff --git a/.azure-pipelines/scripts/models/generate_report.sh b/.azure-pipelines/scripts/models/generate_report.sh index 51b0e3423c8..e76cba525f4 100644 --- a/.azure-pipelines/scripts/models/generate_report.sh +++ b/.azure-pipelines/scripts/models/generate_report.sh @@ -24,52 +24,24 @@ while [[ $# -gt 0 ]];do key=${1} case ${key} in -w|--WORKSPACE) - WORKSPACE=${2} - shift 2 - ;; + WORKSPACE=${2} + shift 2 + ;; --script_path) - script_path=${2} - shift 2 - ;; + script_path=${2} + shift 2 + ;; --output_dir) - output_dir=${2} - shift 2 - ;; + output_dir=${2} + shift 2 + ;; --last_logt_dir) - last_logt_dir=${2} - shift 2 - ;; - --ghprbPullId) - ghprbPullId=${2} - shift 2 - ;; - --MR_source_branch) - MR_source_branch=${2} - shift 2 - ;; - --MR_source_repo) - MR_source_repo=${2} - shift 2 - ;; - --MR_target_branch) - MR_target_branch=${2} - shift 2 - ;; - --repo_url) - repo_url=${2} - shift 2 - ;; - --source_commit_id) - source_commit_id=${2} - shift 2 - ;; - --build_id) - build_id=${2} - shift 2 - ;; + last_logt_dir=${2} + shift 2 + ;; *) - shift - ;; + shift + ;; esac done @@ -86,11 +58,19 @@ summaryLogLast="${last_logt_dir}/summary.log" tuneLogLast="${last_logt_dir}/tuning_info.log" echo "summaryLogLast: ${summaryLogLast}" echo "tuneLogLast: ${tuneLogLast}" +ghprbPullId=${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER} +MR_source_branch=${SYSTEM_PULLREQUEST_SOURCEBRANCH} +MR_source_repo=${SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI} +MR_target_branch=${SYSTEM_PULLREQUEST_TARGETBRANCH} +repo_url=${BUILD_REPOSITORY_URI} +source_commit_id=${BUILD_SOURCEVERSION} +build_id=${BUILD_BUILDID} echo "MR_source_branch: ${MR_source_branch}" echo "MR_source_repo: ${MR_source_repo}" echo "MR_target_branch: ${MR_target_branch}" echo "repo_url: ${repo_url}" echo "commit_id: ${source_commit_id}" +echo "ghprbPullId: ${ghprbPullId}" function main { @@ -98,7 +78,6 @@ function main { generate_html_body generate_results generate_html_footer - } function generate_inference { @@ -205,33 +184,33 @@ function generate_html_core { if(metric == "acc") { target = (int8_result - fp32_result) / fp32_result; if(target >= -0.01) { - printf("%.2f %", target*100); + printf("%.2f %", target*100); }else if(target < -0.05) { - printf("%.2f %", target*100); - job_status = "fail" + printf("%.2f %", target*100); + job_status = "fail" }else{ - printf("%.2f %", target*100); + printf("%.2f %", target*100); } }else if(metric == "perf") { target = int8_result / fp32_result; if(target >= 1.5) { - printf("%.2f", target); + printf("%.2f", target); }else if(target < 1) { - printf("%.2f", target); - job_status = "fail" + printf("%.2f", target); + job_status = "fail" }else{ - printf("%.2f", target); + printf("%.2f", target); } } else { target = int8_result / fp32_result; if(target >= 2) { - printf("%.2f", target); + printf("%.2f", target); }else if(target < 1) { - printf("%.2f", target); - job_status = "fail" + printf("%.2f", target); + job_status = "fail" }else{ - printf("%.2f", target); + printf("%.2f", target); } } }else { @@ -263,14 +242,14 @@ function generate_html_core { } } else { if(new_result == nan && previous_result == nan){ - printf(""); + printf(""); } else{ if(new_result == nan) { - job_status = "fail" - status_png = "background-color:#FFD2D2"; - printf("", status_png); + job_status = "fail" + status_png = "background-color:#FFD2D2"; + printf("", status_png); } else{ - printf(""); + printf(""); } } } @@ -367,7 +346,7 @@ function generate_html_core { printf("\n"); } END{ - printf("\n%s", job_status); + printf("\n%s", job_status); } ' >> ${output_dir}/report.html job_state=$(tail -1 ${WORKSPACE}/report.html) @@ -426,14 +405,14 @@ Test_Info='' if [ "${qtools_branch}" == "" ]; then - commit_id=$(echo ${ghprbActualCommit} |awk '{print substr($1,1,7)}') + commit_id=$(echo ${ghprbActualCommit} |awk '{print substr($1,1,7)}') - MR_TITLE="[ PR-${ghprbPullId} ]" - Test_Info_Title="Source Branch Target Branch Commit " - Test_Info="${MR_source_branch} ${MR_target_branch} ${source_commit_id:0:6}" + MR_TITLE="[ PR-${ghprbPullId} ]" + Test_Info_Title="Source Branch Target Branch Commit " + Test_Info="${MR_source_branch} ${MR_target_branch} ${source_commit_id:0:6}" else - Test_Info_Title="Test Branch Commit ID " - Test_Info="${qtools_branch} ${qtools_commit} " + Test_Info_Title="Test Branch Commit ID " + Test_Info="${qtools_branch} ${qtools_commit} " fi cat >> ${output_dir}/report.html << eof @@ -441,18 +420,20 @@ cat >> ${output_dir}/report.html << eof

Neural Compressor Tuning Tests ${MR_TITLE} - [ Job-${build_id} ]

-

Test Status: ${Jenkins_job_status}

+ [ Job-${build_id} + ] +

Test Status: ${Jenkins_job_status}

Summary

- - ${Test_Info_Title} - - - - ${Test_Info} - + + ${Test_Info_Title} + + + + ${Test_Info} +
Repo
neural-compressor
Repo
neural-compressor
eof @@ -537,68 +518,68 @@ cat > ${output_dir}/report.html << eof } .features-table { - width: 100%; - margin: 0 auto; - border-collapse: separate; - border-spacing: 0; - text-shadow: 0 1px 0 #fff; - color: #2a2a2a; - background: #fafafa; - background-image: -moz-linear-gradient(top, #fff, #eaeaea, #fff); /* Firefox 3.6 */ - background-image: -webkit-gradient(linear,center bottom,center top,from(#fff),color-stop(0.5, #eaeaea),to(#fff)); - font-family: Verdana,Arial,Helvetica + width: 100%; + margin: 0 auto; + border-collapse: separate; + border-spacing: 0; + text-shadow: 0 1px 0 #fff; + color: #2a2a2a; + background: #fafafa; + background-image: -moz-linear-gradient(top, #fff, #eaeaea, #fff); /* Firefox 3.6 */ + background-image: -webkit-gradient(linear,center bottom,center top,from(#fff),color-stop(0.5, #eaeaea),to(#fff)); + font-family: Verdana,Arial,Helvetica } .features-table th,td { - text-align: center; - height: 25px; - line-height: 25px; - padding: 0 8px; - border: 1px solid #cdcdcd; - box-shadow: 0 1px 0 white; - -moz-box-shadow: 0 1px 0 white; - -webkit-box-shadow: 0 1px 0 white; - white-space: nowrap; + text-align: center; + height: 25px; + line-height: 25px; + padding: 0 8px; + border: 1px solid #cdcdcd; + box-shadow: 0 1px 0 white; + -moz-box-shadow: 0 1px 0 white; + -webkit-box-shadow: 0 1px 0 white; + white-space: nowrap; } .no-border th { - box-shadow: none; - -moz-box-shadow: none; - -webkit-box-shadow: none; + box-shadow: none; + -moz-box-shadow: none; + -webkit-box-shadow: none; } .col-cell { - text-align: center; - width: 150px; - font: normal 1em Verdana, Arial, Helvetica; + text-align: center; + width: 150px; + font: normal 1em Verdana, Arial, Helvetica; } .col-cell3 { - background: #efefef; - background: rgba(144,144,144,0.15); + background: #efefef; + background: rgba(144,144,144,0.15); } .col-cell1, .col-cell2 { - background: #B0C4DE; - background: rgba(176,196,222,0.3); + background: #B0C4DE; + background: rgba(176,196,222,0.3); } .col-cellh { - font: bold 1.3em 'trebuchet MS', 'Lucida Sans', Arial; - -moz-border-radius-topright: 10px; - -moz-border-radius-topleft: 10px; - border-top-right-radius: 10px; - border-top-left-radius: 10px; - border-top: 1px solid #eaeaea !important; + font: bold 1.3em 'trebuchet MS', 'Lucida Sans', Arial; + -moz-border-radius-topright: 10px; + -moz-border-radius-topleft: 10px; + border-top-right-radius: 10px; + border-top-left-radius: 10px; + border-top: 1px solid #eaeaea !important; } .col-cellf { - font: bold 1.4em Georgia; - -moz-border-radius-bottomright: 10px; - -moz-border-radius-bottomleft: 10px; - border-bottom-right-radius: 10px; - border-bottom-left-radius: 10px; - border-bottom: 1px solid #dadada !important; + font: bold 1.4em Georgia; + -moz-border-radius-bottomright: 10px; + -moz-border-radius-bottomleft: 10px; + border-bottom-right-radius: 10px; + border-bottom-left-radius: 10px; + border-bottom: 1px solid #dadada !important; } diff --git a/.azure-pipelines/scripts/models/run_benchmark_common.sh b/.azure-pipelines/scripts/models/run_benchmark_common.sh index b342ac48992..ae5b8a1af36 100644 --- a/.azure-pipelines/scripts/models/run_benchmark_common.sh +++ b/.azure-pipelines/scripts/models/run_benchmark_common.sh @@ -1,11 +1,12 @@ #!/bin/bash -set -x +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh # get parameters PATTERN='[-a-zA-Z0-9_]*=' +SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" -for i in "$@" -do +for i in "$@"; do case $i in --framework=*) framework=`echo $i | sed "s/${PATTERN}//"`;; @@ -15,76 +16,114 @@ do input_model=`echo $i | sed "s/${PATTERN}//"`;; --benchmark_cmd=*) benchmark_cmd=`echo $i | sed "s/${PATTERN}//"`;; - --tune_acc=*) - tune_acc=`echo $i | sed "s/${PATTERN}//"`;; --log_dir=*) log_dir=`echo $i | sed "s/${PATTERN}//"`;; --new_benchmark=*) new_benchmark=`echo $i | sed "s/${PATTERN}//"`;; --precision=*) precision=`echo $i | sed "s/${PATTERN}//"`;; + --stage=*) + stage=`echo $i | sed "s/${PATTERN}//"`;; *) echo "Parameter $i not recognized."; exit 1;; esac done -echo "-------- run_benchmark_common --------" +$BOLD_YELLOW && echo "-------- run_benchmark_common --------" && $RESET -# run accuracy -# tune_acc==true means using accuracy results from tuning log -if [ "${tune_acc}" == "false" ]; then - echo "run tuning accuracy in precision ${precision}" - eval "${benchmark_cmd} --input_model=${input_model} --mode=accuracy" 2>&1 | tee ${log_dir}/${framework}-${model}-accuracy-${precision}.log -fi +main() { + # run accuracy + # USE_TUNE_ACC==true means using accuracy results from tuning log + if [ "${USE_TUNE_ACC}" == "false" ]; then + run_accuracy + fi + + # run performance + if [ "${PERF_STABLE_CHECK}" == "false" ]; then + run_performance + else + max_loop=3 + for ((iter = 0; iter < ${max_loop}; iter++)); do + run_performance + { + check_perf_gap + exit_code=$? + } || true + if [ ${exit_code} -ne 0 ]; then + $BOLD_RED && echo "FAILED with performance gap!!" && $RESET + else + $BOLD_GREEN && echo "SUCCEED!!" && $RESET + break + fi + done + exit ${exit_code} + fi +} + +function check_perf_gap() { + python -u ${SCRIPTS_PATH}/collect_log_model.py \ + --framework=${framework} \ + --fwk_ver=${fwk_ver} \ + --model=${model} \ + --logs_dir="${log_dir}" \ + --output_dir="${log_dir}" \ + --build_id=${BUILD_BUILDID} \ + --stage=${stage} +} + +function run_performance() { + cmd="${benchmark_cmd} --input_model=${input_model}" + if [ "${new_benchmark}" == "true" ]; then + $BOLD_YELLOW && echo "run with internal benchmark..." && $RESET + eval ${cmd} 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log + else + $BOLD_YELLOW && echo "run with external multiInstance benchmark..." && $RESET + multiInstance + fi +} + +function run_accuracy() { + $BOLD_YELLOW && echo "run tuning accuracy in precision ${precision}" && $RESET + eval "${benchmark_cmd} --input_model=${input_model} --mode=accuracy" 2>&1 | tee ${log_dir}/${framework}-${model}-accuracy-${precision}.log +} function multiInstance() { - ncores_per_socket=${ncores_per_socket:=$( lscpu | grep 'Core(s) per socket' | cut -d: -f2 | xargs echo -n)} - echo "Executing multi instance benchmark" + ncores_per_socket=${ncores_per_socket:=$(lscpu | grep 'Core(s) per socket' | cut -d: -f2 | xargs echo -n)} + $BOLD_YELLOW && echo "Executing multi instance benchmark" && $RESET ncores_per_instance=4 - echo "ncores_per_socket=${ncores_per_socket}, ncores_per_instance=${ncores_per_instance}" + $BOLD_YELLOW && echo "ncores_per_socket=${ncores_per_socket}, ncores_per_instance=${ncores_per_instance}" && $RESET logFile="${log_dir}/${framework}-${model}-performance-${precision}" benchmark_pids=() - for((j=0;$j<${ncores_per_socket};j=$(($j + ${ncores_per_instance})))); - do - end_core_num=$((j + ncores_per_instance -1)) - if [ ${end_core_num} -ge ${ncores_per_socket} ]; then - end_core_num=$((ncores_per_socket-1)) - fi - numactl -m 0 -C "${j}-${end_core_num}" ${cmd} 2>&1 | tee ${logFile}-${ncores_per_socket}-${ncores_per_instance}-${j}.log & - benchmark_pids+=($!) + for ((j = 0; $j < ${ncores_per_socket}; j = $(($j + ${ncores_per_instance})))); do + end_core_num=$((j + ncores_per_instance - 1)) + if [ ${end_core_num} -ge ${ncores_per_socket} ]; then + end_core_num=$((ncores_per_socket - 1)) + fi + numactl -m 0 -C "${j}-${end_core_num}" ${cmd} 2>&1 | tee ${logFile}-${ncores_per_socket}-${ncores_per_instance}-${j}.log & + benchmark_pids+=($!) done status="SUCCESS" for pid in "${benchmark_pids[@]}"; do wait $pid exit_code=$? - echo "Detected exit code: ${exit_code}" + $BOLD_YELLOW && echo "Detected exit code: ${exit_code}" && $RESET if [ ${exit_code} == 0 ]; then - echo "Process ${pid} succeeded" + $BOLD_GREEN && echo "Process ${pid} succeeded" && $RESET else - echo "Process ${pid} failed" + $BOLD_RED && echo "Process ${pid} failed" && $RESET status="FAILURE" fi done - echo "Benchmark process status: ${status}" + $BOLD_YELLOW && echo "Benchmark process status: ${status}" && $RESET if [ ${status} == "FAILURE" ]; then - echo "Benchmark process returned non-zero exit code." + $BOLD_RED && echo "Benchmark process returned non-zero exit code." && $RESET exit 1 fi } - -# run performance -cmd="${benchmark_cmd} --input_model=${input_model}" - -if [ "${new_benchmark}" == "true" ]; then - echo "run with internal benchmark..." - eval ${cmd} 2>&1 | tee ${log_dir}/${framework}-${model}-performance-${precision}.log -else - echo "run with external multiInstance benchmark..." - multiInstance -fi +main diff --git a/.azure-pipelines/scripts/models/run_model_trigger_common.sh b/.azure-pipelines/scripts/models/run_model_trigger_common.sh index 397ccade727..d0c89560416 100644 --- a/.azure-pipelines/scripts/models/run_model_trigger_common.sh +++ b/.azure-pipelines/scripts/models/run_model_trigger_common.sh @@ -1,5 +1,6 @@ #!/bin/bash - +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh # get parameters PATTERN='[-a-zA-Z0-9_]*=' @@ -32,10 +33,8 @@ do tuning_cmd=`echo $i | sed "s/${PATTERN}//"`;; --benchmark_cmd=*) benchmark_cmd=`echo $i | sed "s/${PATTERN}//"`;; - --tune_acc=*) - tune_acc=`echo $i | sed "s/${PATTERN}//"`;; - --build_id=*) - build_id=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; *) echo "Parameter $i not recognized."; exit 1;; esac @@ -44,113 +43,87 @@ done log_dir="/neural-compressor/.azure-pipelines/scripts/models" WORK_SOURCE_DIR="/neural-compressor/examples/${framework}" SCRIPTS_PATH="/neural-compressor/.azure-pipelines/scripts/models" -echo "processing ${framework}-${fwk_ver}-${model}" -echo "tuning_cmd is ${tuning_cmd}" -echo "benchmark_cmd is ${benchmark_cmd}" - -echo "======= creat log_dir =========" -if [ -d "${log_dir}/${model}" ]; then - echo "${log_dir}/${model} already exists, don't need to mkdir." -else - echo "no log dir ${log_dir}/${model}, create." - cd ${log_dir} - mkdir ${model} -fi - -echo "====== install requirements ======" -/bin/bash /neural-compressor/.azure-pipelines/scripts/install_nc.sh - -cd ${WORK_SOURCE_DIR}/${model_src_dir} -pip install ruamel_yaml -pip install psutil -pip install protobuf==3.20.1 -if [[ "${framework}" == "tensorflow" ]]; then - pip install intel-tensorflow==${fwk_ver} -elif [[ "${framework}" == "pytorch" ]]; then - pip install torch==${fwk_ver} -f https://download.pytorch.org/whl/torch_stable.html - pip install torchvision==${torch_vision_ver} -f https://download.pytorch.org/whl/torch_stable.html -elif [[ "${framework}" == "onnxrt" ]]; then - pip install onnx==1.11.0 - pip install onnxruntime==${fwk_ver} -elif [[ "${framework}" == "mxnet" ]]; then - if [[ "${fwk_ver}" == "1.7.0" ]]; then - pip install mxnet==${fwk_ver}.post2 - elif [[ "${fwk_ver}" == "1.6.0" ]]; then - pip install mxnet-mkl==${mxnet_version} - else - pip install mxnet==${fwk_ver} - fi -fi - -if [ -f "requirements.txt" ]; then - sed -i '/neural-compressor/d' requirements.txt - if [ "${framework}" == "onnxrt" ]; then - sed -i '/^onnx>=/d;/^onnx==/d;/^onnxruntime>=/d;/^onnxruntime==/d' requirements.txt - fi - if [ "${framework}" == "tensorflow" ]; then - sed -i '/tensorflow==/d;/tensorflow$/d' requirements.txt - sed -i '/^intel-tensorflow/d' requirements.txt +$BOLD_YELLOW && echo "processing ${framework}-${fwk_ver}-${model}" && $RESET +$BOLD_YELLOW && echo "benchmark_cmd is ${benchmark_cmd}" && $RESET + +if [ "${mode}" == "env_setup" ]; then + /bin/bash env_setup.sh \ + --yaml=${yaml} \ + --framework=${framework} \ + --fwk_ver=${fwk_ver} \ + --torch_vision_ver=${torch_vision_ver} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} +elif [ "${mode}" == "tuning" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "tuning_cmd is ${tuning_cmd}" && $RESET + $BOLD_YELLOW && echo "======== run tuning ========" && $RESET + /bin/bash ${SCRIPTS_PATH}/run_tuning_common.sh \ + --framework=${framework} \ + --model=${model} \ + --tuning_cmd="${tuning_cmd}" \ + --log_dir="${log_dir}/${model}" \ + --input_model=${input_model} \ + --strategy=${strategy} \ + 2>&1 | tee -a ${log_dir}/${model}/${framework}-${model}-tune.log + $BOLD_YELLOW && echo "====== check tuning status. ======" && $RESET + control_phrase="model which meet accuracy goal." + if [ $(grep "${control_phrase}" ${log_dir}/${model}/${framework}-${model}-tune.log | wc -l) == 0 ];then + $BOLD_RED && echo "====== tuning FAILED!! ======" && $RESET; exit 1 fi - if [ "${framework}" == "mxnet" ]; then - sed -i '/mxnet==/d;/mxnet$/d;/mxnet-mkl==/d;/mxnet-mkl$/d' requirements.txt + if [ $(grep "${control_phrase}" ${log_dir}/${model}/${framework}-${model}-tune.log | grep "Not found" | wc -l) == 1 ];then + $BOLD_RED && echo "====== tuning FAILED!! ======" && $RESET; exit 1 fi - if [ "${framework}" == "pytorch" ]; then - sed -i '/torch==/d;/torch$/d;/torchvision==/d;/torchvision$/d' requirements.txt + $BOLD_GREEN && echo "====== tuning SUCCEED!! ======" && $RESET +elif [ "${mode}" == "fp32_benchmark" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "benchmark_cmd is ${benchmark_cmd}" && $RESET + $BOLD_YELLOW && echo "====== run benchmark fp32 =======" && $RESET + /bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh \ + --framework=${framework} \ + --model=${model} \ + --input_model=${input_model} \ + --benchmark_cmd="${benchmark_cmd}" \ + --log_dir="${log_dir}/${model}" \ + --new_benchmark=${new_benchmark} \ + --precision="fp32" \ + --stage=${mode} +elif [ "${mode}" == "int8_benchmark" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "benchmark_cmd is ${benchmark_cmd}" && $RESET + $BOLD_YELLOW && echo "====== run benchmark int8 =======" && $RESET + if [[ "${framework}" == "onnxrt" ]]; then + model_name="${log_dir}/${model}/${framework}-${model}-tune.onnx" + elif [[ "${framework}" == "mxnet" ]]; then + model_name="${log_dir}/${model}" + elif [[ "${framework}" == "tensorflow" ]]; then + model_name="${log_dir}/${model}/${framework}-${model}-tune.pb" + elif [[ "${framework}" == "pytorch" ]]; then + model_name=${input_model} + benchmark_cmd="${benchmark_cmd} --int8=true" fi - n=0 - until [ "$n" -ge 5 ] - do - python -m pip install -r requirements.txt && break - n=$((n+1)) - sleep 5 - done - pip list -else - echo "Not found requirements.txt file." + /bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh \ + --framework=${framework} \ + --model=${model} \ + --input_model="${model_name}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --log_dir="${log_dir}/${model}" \ + --new_benchmark=${new_benchmark} \ + --precision="int8" \ + --stage=${mode} +elif [ "${mode}" == "collect_log" ]; then + cd ${WORK_SOURCE_DIR}/${model_src_dir} + $BOLD_YELLOW && echo "====== collect logs of model ${model} =======" && $RESET + python -u ${SCRIPTS_PATH}/collect_log_model.py \ + --framework=${framework} \ + --fwk_ver=${fwk_ver} \ + --model=${model} \ + --logs_dir="${log_dir}/${model}" \ + --output_dir="${log_dir}/${model}" \ + --build_id=${BUILD_BUILDID} + $BOLD_YELLOW && echo "====== Finish collect logs =======" && $RESET fi - - -echo "======== update yaml config ========" -echo -e "\nPrint origin yaml..." -cat ${yaml} -python ${SCRIPTS_PATH}/update_yaml_config.py --yaml=${yaml} --framework=${framework} \ ---dataset_location=${dataset_location} --batch_size=${batch_size} --strategy=${strategy} \ ---new_benchmark=${new_benchmark} --multi_instance='true' -echo -e "\nPrint updated yaml... " -cat ${yaml} - - -echo "======== run tuning ========" -/bin/bash ${SCRIPTS_PATH}/run_tuning_common.sh --framework=${framework} --model=${model} \ ---tuning_cmd="${tuning_cmd}" --log_dir="${log_dir}/${model}" --input_model=${input_model} --strategy=${strategy} \ -2>&1 | tee -a ${log_dir}/${model}/${framework}-${model}-tune.log - - -echo "====== run benchmark fp32 =======" -/bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh --framework=${framework} --model=${model} \ - --input_model=${input_model} --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} \ - --log_dir="${log_dir}/${model}" --new_benchmark=${new_benchmark} --precision="fp32" - - -echo "====== run benchmark int8 =======" -if [[ "${framework}" == "onnxrt" ]]; then - model_name="${log_dir}/${model}/${framework}-${model}-tune.onnx" -elif [[ "${framework}" == "mxnet" ]]; then - model_name="${log_dir}/${model}" -elif [[ "${framework}" == "tensorflow" ]]; then - model_name="${log_dir}/${model}/${framework}-${model}-tune.pb" -elif [[ "${framework}" == "pytorch" ]]; then - model_name=${input_model} - benchmark_cmd="${benchmark_cmd} --int8=true" -fi -/bin/bash ${SCRIPTS_PATH}/run_benchmark_common.sh --framework=${framework} --model=${model} \ - --input_model="${model_name}" --benchmark_cmd="${benchmark_cmd}" \ - --tune_acc=${tune_acc} --log_dir="${log_dir}/${model}" --new_benchmark=${new_benchmark} --precision="int8" - - -echo "====== collect logs of model ${model} =======" -python -u ${SCRIPTS_PATH}/collect_log_model.py --framework=${framework} --fwk_ver=${fwk_ver} --model=${model} \ ---logs_dir="${log_dir}/${model}" --output_dir="${log_dir}/${model}" --build_id=${build_id} - - -echo "====== Finish model test =======" \ No newline at end of file diff --git a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh index 455ee809c61..8bf3b293fc2 100644 --- a/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_mxnet_models_trigger.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -eo pipefail # get parameters PATTERN='[-a-zA-Z0-9_]*=' @@ -8,10 +8,8 @@ do case $i in --model=*) model=`echo $i | sed "s/${PATTERN}//"`;; - --tune_acc=*) - tune_acc=`echo $i | sed "s/${PATTERN}//"`;; - --build_id=*) - build_id=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; *) echo "Parameter $i not recognized."; exit 1;; esac @@ -35,7 +33,17 @@ if [ "${model}" == "resnet50v1" ]; then fi -/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ ---model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ ---input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ ---tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} --build_id=${build_id} +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh index d71081f61b3..a69852f01a5 100644 --- a/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_onnxrt_models_trigger.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -eo pipefail # get parameters PATTERN='[-a-zA-Z0-9_]*=' @@ -8,10 +8,8 @@ do case $i in --model=*) model=`echo $i | sed "s/${PATTERN}//"`;; - --tune_acc=*) - tune_acc=`echo $i | sed "s/${PATTERN}//"`;; - --build_id=*) - build_id=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; *) echo "Parameter $i not recognized."; exit 1;; esac @@ -31,7 +29,7 @@ if [ "${model}" == "resnet50-v1-12" ]; then batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" - benchmark_cmd="bash run_benchmark.sh --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" elif [ "${model}" == "bert_base_MRPC_static" ]; then model_src_dir="language_translation/bert/quantization/ptq" dataset_location="/tf_dataset/pytorch/glue_data/MRPC" @@ -41,7 +39,7 @@ elif [ "${model}" == "bert_base_MRPC_static" ]; then batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" - benchmark_cmd="bash run_benchmark.sh --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" elif [ "${model}" == "bert_base_MRPC_dynamic" ]; then model_src_dir="language_translation/bert/quantization/ptq" dataset_location="/tf_dataset/pytorch/glue_data/MRPC" @@ -51,7 +49,7 @@ elif [ "${model}" == "bert_base_MRPC_dynamic" ]; then batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" - benchmark_cmd="bash run_benchmark.sh --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" elif [ "${model}" == "distilbert_base_MRPC_qdq" ]; then model_src_dir="language_translation/distilbert/quantization/ptq" dataset_location="/tf_dataset/pytorch/glue_data/MRPC" @@ -61,11 +59,21 @@ elif [ "${model}" == "distilbert_base_MRPC_qdq" ]; then batch_size=1 new_benchmark=true tuning_cmd="bash run_tuning.sh --input_model=${input_model} --config=${yaml}" - benchmark_cmd="bash run_benchmark.sh --config=${yaml}" + benchmark_cmd="bash run_benchmark.sh --config=${yaml} --mode=performance" fi -/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ ---model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ ---input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ ---tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd} --mode=performance" --tune_acc=${tune_acc} --build_id=${build_id} +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh index 9aef437666d..5cd776816f4 100644 --- a/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_pytorch_models_trigger.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -eo pipefail # get parameters PATTERN='[-a-zA-Z0-9_]*=' @@ -8,10 +8,8 @@ do case $i in --model=*) model=`echo $i | sed "s/${PATTERN}//"`;; - --tune_acc=*) - tune_acc=`echo $i | sed "s/${PATTERN}//"`;; - --build_id=*) - build_id=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; *) echo "Parameter $i not recognized."; exit 1;; esac @@ -46,8 +44,18 @@ elif [ "${model}" == "resnet18_fx" ]; then fi -/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ ---torch_vision_ver=${TORCH_VISION_VERSION} --model=${model} --model_src_dir=${model_src_dir} \ ---dataset_location=${dataset_location} --input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} \ ---new_benchmark=${new_benchmark} --tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" \ ---tune_acc=${tune_acc} --build_id=${build_id} +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --torch_vision_ver=${TORCH_VISION_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh index 1b3a6c6815d..b3eee910900 100644 --- a/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh +++ b/.azure-pipelines/scripts/models/run_tensorflow_models_trigger.sh @@ -1,5 +1,5 @@ #!/bin/bash - +set -eo pipefail # get parameters PATTERN='[-a-zA-Z0-9_]*=' @@ -8,10 +8,8 @@ do case $i in --model=*) model=`echo $i | sed "s/${PATTERN}//"`;; - --tune_acc=*) - tune_acc=`echo $i | sed "s/${PATTERN}//"`;; - --build_id=*) - build_id=`echo $i | sed "s/${PATTERN}//"`;; + --mode=*) + mode=`echo $i | sed "s/${PATTERN}//"`;; *) echo "Parameter $i not recognized."; exit 1;; esac @@ -104,7 +102,17 @@ elif [ "${model}" == "resnet50_fashion" ]; then fi -/bin/bash run_model_trigger_common.sh --yaml=${yaml} --framework=${FRAMEWORK} --fwk_ver=${FRAMEWORK_VERSION} \ ---model=${model} --model_src_dir=${model_src_dir} --dataset_location=${dataset_location} \ ---input_model=${input_model} --batch_size=${batch_size} --strategy=${strategy} --new_benchmark=${new_benchmark} \ ---tuning_cmd="${tuning_cmd}" --benchmark_cmd="${benchmark_cmd}" --tune_acc=${tune_acc} --build_id=${build_id} +/bin/bash run_model_trigger_common.sh \ + --yaml=${yaml} \ + --framework=${FRAMEWORK} \ + --fwk_ver=${FRAMEWORK_VERSION} \ + --model=${model} \ + --model_src_dir=${model_src_dir} \ + --dataset_location=${dataset_location} \ + --input_model=${input_model} \ + --batch_size=${batch_size} \ + --strategy=${strategy} \ + --new_benchmark=${new_benchmark} \ + --tuning_cmd="${tuning_cmd}" \ + --benchmark_cmd="${benchmark_cmd}" \ + --mode=${mode} diff --git a/.azure-pipelines/scripts/models/run_tuning_common.sh b/.azure-pipelines/scripts/models/run_tuning_common.sh index 26b223bf994..fbb68d65605 100644 --- a/.azure-pipelines/scripts/models/run_tuning_common.sh +++ b/.azure-pipelines/scripts/models/run_tuning_common.sh @@ -1,5 +1,6 @@ #!/bin/bash -set -x +set -eo pipefail +source /neural-compressor/.azure-pipelines/scripts/change_color.sh # get parameters PATTERN='[-a-zA-Z0-9_]*=' @@ -35,16 +36,15 @@ else output_model=${log_dir}/${framework}-${model}-tune.pb fi -echo -e "-------- run_tuning_common --------" -echo ${tuning_cmd} +$BOLD_YELLOW && echo -e "-------- run_tuning_common --------" && $RESET +$BOLD_YELLOW && echo ${tuning_cmd} && $RESET + eval "/usr/bin/time -v ${tuning_cmd} --output_model=${output_model}" -echo "====== finish tuning. echo information. ======" +$BOLD_YELLOW && echo "====== finish tuning. echo information. ======" && $RESET endtime=`date +'%Y-%m-%d %H:%M:%S'` start_seconds=$(date --date="$starttime" +%s); end_seconds=$(date --date="$endtime" +%s); -echo "Tuning time spend: "$((end_seconds-start_seconds))"s " - -echo "Tuning strategy: ${strategy}" - -echo "Total resident size (kbytes): $(cat /proc/meminfo |grep 'MemTotal' |sed 's/[^0-9]//g')" +$BOLD_GREEN && echo "Tuning time spend: "$((end_seconds-start_seconds))"s " && $RESET +$BOLD_GREEN && echo "Tuning strategy: ${strategy}" && $RESET +$BOLD_GREEN && echo "Total resident size (kbytes): $(cat /proc/meminfo | grep 'MemTotal' | sed 's/[^0-9]//g')" && $RESET diff --git a/.azure-pipelines/scripts/models/update_yaml_config.py b/.azure-pipelines/scripts/models/update_yaml_config.py index 9d92a457f6d..c305134e18d 100644 --- a/.azure-pipelines/scripts/models/update_yaml_config.py +++ b/.azure-pipelines/scripts/models/update_yaml_config.py @@ -64,9 +64,9 @@ def update_yaml_dataset(yaml, framework, dataset_location): line = re.sub(key_patterns["pattern"], key_patterns["replacement"], line) config.write(line) - if framework == "pytorch": - val_dataset = dataset_location + f"\{os.path.sep}" + "val" - train_dataset = dataset_location + f"\{os.path.sep}" + "train" + else: + val_dataset = dataset_location + f"{os.path.sep}" + "val" + train_dataset = dataset_location + f"{os.path.sep}" + "train" patterns = { "calibration_dataset": { "pattern": r'root:.*/path/to/calibration/dataset/?', @@ -78,14 +78,6 @@ def update_yaml_dataset(yaml, framework, dataset_location): }, } - if topology == "distilbert_base_MRPC": - patterns.update({ - "bert_name": { - "pattern": r'name:/s+bert', - "replacement": "name: distilbert", - } - }) - print("======= update_yaml_dataset =======") with open(yaml, "w") as config: for line in lines: diff --git a/.azure-pipelines/template/model-template.yml b/.azure-pipelines/template/model-template.yml index 1bc9dd4a7bf..f145025faf2 100644 --- a/.azure-pipelines/template/model-template.yml +++ b/.azure-pipelines/template/model-template.yml @@ -13,18 +13,56 @@ parameters: steps: - template: docker-template.yml parameters: - dockerConfigName: 'commonDockerConfig' - repoName: 'neural-compressor' - repoTag: 'py38' - dockerFileName: 'Dockerfile' + dockerConfigName: "commonDockerConfig" + repoName: "neural-compressor" + repoTag: "py38" + dockerFileName: "Dockerfile" containerName: ${{ parameters.modelContainerName }} - script: | docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ - && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --tune_acc=true --build_id=$(Build.BuildId)" - displayName: Tune&Benchmark ${{ parameters.modelName }} + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='env_setup'" + displayName: Env setup + + - task: DownloadPipelineArtifact@2 + inputs: + source: "specific" + artifact: ${{ parameters.framework }}_${{ parameters.modelName }} + patterns: "**_summary.log" + path: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}_refer_log + project: $(System.TeamProject) + pipeline: "Model-Test" + runVersion: "specific" + runId: $(refer_buildId) + retryDownloadCount: 3 + displayName: "Download refer logs" + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='tuning'" + displayName: Tuning + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='int8_benchmark'" + displayName: INT8 Benchmark + + - script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='fp32_benchmark'" + displayName: FP32 Benchmark + + - task: Bash@3 + condition: always() + inputs: + targetType: "inline" + script: | + docker exec ${{ parameters.modelContainerName }} bash -c "cd /neural-compressor/.azure-pipelines/scripts/models \ + && bash run_${{ parameters.framework }}_models_trigger.sh --model=${{ parameters.modelName }} --mode='collect_log'" + displayName: Collect log - task: PublishPipelineArtifact@1 + condition: always() inputs: targetPath: $(Build.SourcesDirectory)/.azure-pipelines/scripts/models/${{ parameters.modelName }}/ artifact: ${{ parameters.framework }}_${{ parameters.modelName }}