From 2f874ccd82c1543aee2e30783bbc4dd503a1dc6b Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 20 Oct 2025 10:16:17 +0800 Subject: [PATCH 1/7] Add compatibility test pipeline and update unit test configurations Signed-off-by: Sun, Xuehao --- .azure-pipelines/compatibility-test.yml | 70 +++++++++++++++++++++++ .azure-pipelines/scripts/ut/run_ut.sh | 20 +++++-- .azure-pipelines/template/ut-template.yml | 2 +- .azure-pipelines/unit-test.yml | 2 - 4 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 .azure-pipelines/compatibility-test.yml diff --git a/.azure-pipelines/compatibility-test.yml b/.azure-pipelines/compatibility-test.yml new file mode 100644 index 000000000..f8ec22979 --- /dev/null +++ b/.azure-pipelines/compatibility-test.yml @@ -0,0 +1,70 @@ +trigger: none + +pr: + autoCancel: true + drafts: false + branches: + include: + - main + paths: + include: + - auto_round + - auto_round_extension + - setup.py + - setup.cfg + - requirements.txt + - requirements-cpu.txt + - .azure-pipelines/compatibility-test.yml + exclude: + - "*.md" + - "**/*.md" + +jobs: +- job: Test + displayName: 'Python Import Test' + strategy: + matrix: + Python310_Linux: + python_version: '3.10' + vmImage: 'ubuntu-latest' + Python311_Linux: + python_version: '3.11' + vmImage: 'ubuntu-latest' + Python312_Linux: + python_version: '3.12' + vmImage: 'ubuntu-latest' + Python313_Linux: + python_version: '3.13' + vmImage: 'ubuntu-latest' + + Python310_Windows: + python_version: '3.10' + vmImage: 'windows-latest' + Python311_Windows: + python_version: '3.11' + vmImage: 'windows-latest' + Python312_Windows: + python_version: '3.12' + vmImage: 'windows-latest' + Python313_Windows: + python_version: '3.13' + vmImage: 'windows-latest' + + pool: + vmImage: $(vmImage) + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + displayName: 'Use Python $(python_version)' + + - script: | + python -m pip install --upgrade pip uv + uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu + uv pip install . + displayName: 'Install dependencies' + + - script: | + python -c "import auto_round" + displayName: 'Run compatibility test' diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index cf3997357..112b687c8 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -26,13 +26,21 @@ LOG_DIR=/auto-round/log_dir mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut.log -# Split test files into 5 parts -find . -name "test*.py" ! -name "*hpu_only*.py" | sort > all_tests.txt +# Split test files into 4 parts +find . -name "test*.py" | sort > all_tests.txt total_lines=$(wc -l < all_tests.txt) -NUM_CHUNKS=5 -chunk_size=$(( (total_lines + NUM_CHUNKS - 1) / NUM_CHUNKS )) -start_line=$(( (test_part - 1) * chunk_size + 1 )) -selected_files=$(tail -n +$start_line all_tests.txt | head -n $chunk_size) +NUM_CHUNKS=4 +q=$(( total_lines / NUM_CHUNKS )) +r=$(( total_lines % NUM_CHUNKS )) +if [ "$test_part" -le "$r" ]; then + chunk_size=$(( q + 1 )) + start_line=$(( (test_part - 1) * chunk_size + 1 )) +else + chunk_size=$q + start_line=$(( r * (q + 1) + (test_part - r - 1) * q + 1 )) +fi +end_line=$(( start_line + chunk_size - 1 )) +selected_files=$(sed -n "${start_line},${end_line}p" all_tests.txt) printf '%s\n' "${selected_files}" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run.sh cat run.sh bash run.sh 2>&1 | tee "${ut_log_name}" diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml index 3962f072d..e5a081b0a 100644 --- a/.azure-pipelines/template/ut-template.yml +++ b/.azure-pipelines/template/ut-template.yml @@ -55,7 +55,7 @@ steps: displayName: "Run UT" - task: PublishPipelineArtifact@1 - condition: succeededOrFailed() + condition: succeeded() inputs: targetPath: ${{ parameters.uploadPath }} artifact: ${{ parameters.utArtifact }}_coverage diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index 6a3d1338a..4a74f1204 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -50,8 +50,6 @@ stages: PART: 3 part4: PART: 4 - part5: - PART: 5 steps: - template: template/ut-template.yml parameters: From 229a78dd0297c0b58fd7292e45a702f641895f90 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 20 Oct 2025 10:23:37 +0800 Subject: [PATCH 2/7] fix env Signed-off-by: Sun, Xuehao --- .azure-pipelines/compatibility-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/compatibility-test.yml b/.azure-pipelines/compatibility-test.yml index f8ec22979..4b14ed2c1 100644 --- a/.azure-pipelines/compatibility-test.yml +++ b/.azure-pipelines/compatibility-test.yml @@ -21,7 +21,7 @@ pr: jobs: - job: Test - displayName: 'Python Import Test' + displayName: 'Compatibility Test' strategy: matrix: Python310_Linux: @@ -60,6 +60,9 @@ jobs: displayName: 'Use Python $(python_version)' - script: | + export PYTHONUNBUFFERED=1 + export UV_NO_PROGRESS=1 + export UV_SYSTEM_PYTHON=1 python -m pip install --upgrade pip uv uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu uv pip install . From 9718c852b1c2635d1888f20e6755af23e9af3ec8 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 20 Oct 2025 10:45:32 +0800 Subject: [PATCH 3/7] fix env Signed-off-by: Sun, Xuehao --- .azure-pipelines/compatibility-test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/compatibility-test.yml b/.azure-pipelines/compatibility-test.yml index 4b14ed2c1..91b7e3ad5 100644 --- a/.azure-pipelines/compatibility-test.yml +++ b/.azure-pipelines/compatibility-test.yml @@ -60,12 +60,14 @@ jobs: displayName: 'Use Python $(python_version)' - script: | - export PYTHONUNBUFFERED=1 - export UV_NO_PROGRESS=1 - export UV_SYSTEM_PYTHON=1 python -m pip install --upgrade pip uv uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu uv pip install . + pip list + env: + PYTHONUNBUFFERED: '1' + UV_NO_PROGRESS: '1' + UV_SYSTEM_PYTHON: '1' displayName: 'Install dependencies' - script: | From 9ba3974c2f23ee51c1f4b24cc8107d018d03096f Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 20 Oct 2025 11:03:18 +0800 Subject: [PATCH 4/7] add stages Signed-off-by: Sun, Xuehao --- .azure-pipelines/compatibility-test.yml | 103 ++++++++++++------------ 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/.azure-pipelines/compatibility-test.yml b/.azure-pipelines/compatibility-test.yml index 91b7e3ad5..05ca73fd2 100644 --- a/.azure-pipelines/compatibility-test.yml +++ b/.azure-pipelines/compatibility-test.yml @@ -19,57 +19,60 @@ pr: - "*.md" - "**/*.md" -jobs: -- job: Test - displayName: 'Compatibility Test' - strategy: - matrix: - Python310_Linux: - python_version: '3.10' - vmImage: 'ubuntu-latest' - Python311_Linux: - python_version: '3.11' - vmImage: 'ubuntu-latest' - Python312_Linux: - python_version: '3.12' - vmImage: 'ubuntu-latest' - Python313_Linux: - python_version: '3.13' - vmImage: 'ubuntu-latest' +stages: + - stage: Compatibility Test + dependsOn: [] + jobs: + - job: + timeoutInMinutes: 20 + strategy: + matrix: + Python310_Linux: + python_version: '3.10' + vmImage: 'ubuntu-latest' + Python311_Linux: + python_version: '3.11' + vmImage: 'ubuntu-latest' + Python312_Linux: + python_version: '3.12' + vmImage: 'ubuntu-latest' + Python313_Linux: + python_version: '3.13' + vmImage: 'ubuntu-latest' - Python310_Windows: - python_version: '3.10' - vmImage: 'windows-latest' - Python311_Windows: - python_version: '3.11' - vmImage: 'windows-latest' - Python312_Windows: - python_version: '3.12' - vmImage: 'windows-latest' - Python313_Windows: - python_version: '3.13' - vmImage: 'windows-latest' - - pool: - vmImage: $(vmImage) + Python310_Windows: + python_version: '3.10' + vmImage: 'windows-latest' + Python311_Windows: + python_version: '3.11' + vmImage: 'windows-latest' + Python312_Windows: + python_version: '3.12' + vmImage: 'windows-latest' + Python313_Windows: + python_version: '3.13' + vmImage: 'windows-latest' + + pool: + vmImage: $(vmImage) - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - displayName: 'Use Python $(python_version)' + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python_version)' + displayName: 'Use Python $(python_version)' - - script: | - python -m pip install --upgrade pip uv - uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu - uv pip install . - pip list - env: - PYTHONUNBUFFERED: '1' - UV_NO_PROGRESS: '1' - UV_SYSTEM_PYTHON: '1' - displayName: 'Install dependencies' + - script: | + python -m pip install --upgrade pip uv + uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu + uv pip install . + pip list + env: + PYTHONUNBUFFERED: '1' + UV_NO_PROGRESS: '1' + UV_SYSTEM_PYTHON: '1' + displayName: 'Install dependencies' - - script: | - python -c "import auto_round" - displayName: 'Run compatibility test' + - script: | + python -c "import auto_round" + displayName: 'Run compatibility test' From 6aa43d5112280477bbe10a17cc7c856ae44b8ae9 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 20 Oct 2025 11:04:18 +0800 Subject: [PATCH 5/7] fix name Signed-off-by: Sun, Xuehao --- .azure-pipelines/compatibility-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/compatibility-test.yml b/.azure-pipelines/compatibility-test.yml index 05ca73fd2..3cdb8ace6 100644 --- a/.azure-pipelines/compatibility-test.yml +++ b/.azure-pipelines/compatibility-test.yml @@ -20,7 +20,8 @@ pr: - "**/*.md" stages: - - stage: Compatibility Test + - stage: + displayName: Compatibility Test dependsOn: [] jobs: - job: From 3a950c69c42169bc174ec59cbea656175e8a6a87 Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Mon, 20 Oct 2025 14:48:51 +0800 Subject: [PATCH 6/7] fix parts Signed-off-by: Sun, Xuehao --- .azure-pipelines/scripts/ut/run_ut.sh | 4 ++-- .azure-pipelines/unit-test.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh index 112b687c8..50f8238bd 100644 --- a/.azure-pipelines/scripts/ut/run_ut.sh +++ b/.azure-pipelines/scripts/ut/run_ut.sh @@ -26,10 +26,10 @@ LOG_DIR=/auto-round/log_dir mkdir -p ${LOG_DIR} ut_log_name=${LOG_DIR}/ut.log -# Split test files into 4 parts +# Split test files into 5 parts find . -name "test*.py" | sort > all_tests.txt total_lines=$(wc -l < all_tests.txt) -NUM_CHUNKS=4 +NUM_CHUNKS=5 q=$(( total_lines / NUM_CHUNKS )) r=$(( total_lines % NUM_CHUNKS )) if [ "$test_part" -le "$r" ]; then diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml index 4a74f1204..6a3d1338a 100644 --- a/.azure-pipelines/unit-test.yml +++ b/.azure-pipelines/unit-test.yml @@ -50,6 +50,8 @@ stages: PART: 3 part4: PART: 4 + part5: + PART: 5 steps: - template: template/ut-template.yml parameters: From a5ff23b1e8e12d11fb08f34910379b65d703a5ff Mon Sep 17 00:00:00 2001 From: "Sun, Xuehao" Date: Tue, 21 Oct 2025 14:15:32 +0800 Subject: [PATCH 7/7] update cuda ut Signed-off-by: Sun, Xuehao --- .azure-pipelines/scripts/ut/run_ut_cuda.sh | 112 +++++++++++++++------ 1 file changed, 84 insertions(+), 28 deletions(-) diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh index 92fb15490..a7b6eddd7 100644 --- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh +++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh @@ -5,6 +5,10 @@ CONDA_ENV_NAME="unittest_cuda" PYTHON_VERSION="3.10" REPO_PATH=$(git rev-parse --show-toplevel) LOG_DIR=${REPO_PATH}/ut_log_dir +SUMMARY_LOG=${LOG_DIR}/results_summary.log + +rm -rf ${LOG_DIR} && mkdir -p ${LOG_DIR} +touch ${SUMMARY_LOG} [[ -z "$CUDA_VISIBLE_DEVICES" ]] && export CUDA_VISIBLE_DEVICES=0 function create_conda_env() { @@ -34,6 +38,55 @@ function create_conda_env() { uv pip install pytest-cov pytest-html cmake==4.0.2 } +function print_test_results_table() { + local log_pattern=$1 + local test_type=$2 + + echo "" + echo "==========================================" >> ${SUMMARY_LOG} + echo "Test Results Summary - ${test_type}" >> ${SUMMARY_LOG} + echo "==========================================" >> ${SUMMARY_LOG} + printf "%-30s %-10s %-50s\n" "Test Case" "Result" "Log File" >> ${SUMMARY_LOG} + printf "%-30s %-10s %-50s\n" "----------" "------" "--------" >> ${SUMMARY_LOG} + + local total_tests=0 + local passed_tests=0 + local failed_tests=0 + + for log_file in ${LOG_DIR}/${log_pattern}; do + if [ -f "${log_file}" ]; then + local test_name=$(basename "${log_file}" .log) + # Remove prefix to get clean test case name + test_name=${test_name#unittest_cuda_} + test_name=${test_name#unittest_cuda_vlm_} + + local result="UNKNOWN" + local failure_count=$(grep -c '== FAILURES ==' "${log_file}" 2>/dev/null || echo 0) + local error_count=$(grep -c '== ERRORS ==' "${log_file}" 2>/dev/null || echo 0) + local passed_count=$(grep -c ' passed' "${log_file}" 2>/dev/null || echo 0) + + if [ ${failure_count} -gt 0 ] || [ ${error_count} -gt 0 ]; then + result="FAILED" + failed_tests=$((failed_tests + 1)) + elif [ ${passed_count} -gt 0 ]; then + result="PASSED" + passed_tests=$((passed_tests + 1)) + else + result="NO_TESTS" + fi + + total_tests=$((total_tests + 1)) + local log_filename=$(basename "${log_file}") + printf "%-30s %-10s %-50s\n" "${test_name}" "${result}" "${log_filename}" >> ${SUMMARY_LOG} + fi + done + + echo "==========================================" >> ${SUMMARY_LOG} + printf "Total: %d, Passed: %d, Failed: %d\n" ${total_tests} ${passed_tests} ${failed_tests} >> ${SUMMARY_LOG} + echo "==========================================" >> ${SUMMARY_LOG} + echo "" >> ${SUMMARY_LOG} +} + function run_unit_test() { # install unit test dependencies create_conda_env @@ -49,24 +102,25 @@ function run_unit_test() { uv pip install -r requirements.txt uv pip install -r requirements_diffusion.txt - uv pip list + pip list > ${LOG_DIR}/ut_pip_list.txt export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])') - - # setup test env - mkdir -p ${LOG_DIR} - local ut_log_name=${LOG_DIR}/unittest_cuda.log - find . -name "test_*.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" >run.sh - cat run.sh - - # run unit test - bash run.sh 2>&1 | tee ${ut_log_name} - - cp report.html ${LOG_DIR}/ - cp coverage.xml ${LOG_DIR}/ - - if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then - echo "Find errors in pytest case, please check the output..." + + # run unit tests individually with separate logs + for test_file in $(find . -name "test_*.py"); do + local test_basename=$(basename ${test_file} .py) + local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.log + echo "Running ${test_file}..." + + python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name} + done + + mv report.html ${LOG_DIR}/ + mv coverage.xml ${LOG_DIR}/ + + # Print test results table and check for failures + if ! print_test_results_table "unittest_cuda_test_*.log" "CUDA Unit Tests"; then + echo "Some CUDA unit tests failed. Please check the individual log files for details." fi } @@ -85,30 +139,32 @@ function run_unit_test_vlm() { uv pip install flash-attn==2.7.4.post1 --no-build-isolation uv pip install -r requirements_vlm.txt - uv pip list + pip list > ${LOG_DIR}/vlm_ut_pip_list.txt export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])') - # setup test env - mkdir -p ${LOG_DIR} - local ut_log_name=${LOG_DIR}/unittest_cuda_vlm.log - find . -name "test*vlms.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report_vlms.html --self-contained-html --cov-report xml:coverage_vlms.xml --cov-append -vs --disable-warnings ,g" >run_vlms.sh - cat run_vlms.sh + # run VLM unit tests individually with separate logs + for test_file in $(find . -name "test*vlms.py"); do + local test_basename=$(basename ${test_file} .py) + local ut_log_name=${LOG_DIR}/unittest_cuda_vlm_${test_basename}.log + echo "Running ${test_file}..." - # run unit test - bash run_vlms.sh 2>&1 | tee ${ut_log_name} + python -m pytest --cov="${auto_round_path}" --cov-report term --html=report_vlms.html --self-contained-html --cov-report xml:coverage_vlms.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name} + done - cp report_vlms.html ${LOG_DIR}/ - cp coverage_vlms.xml ${LOG_DIR}/ + mv report_vlms.html ${LOG_DIR}/ + mv coverage_vlms.xml ${LOG_DIR}/ - if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then - echo "Find errors in pytest case, please check the output..." + # Print test results table and check for failures + if ! print_test_results_table "unittest_cuda_vlm_test*.log" "CUDA VLM Tests"; then + echo "Some CUDA VLM tests failed. Please check the individual log files for details." fi } function main() { run_unit_test_vlm run_unit_test + cat ${SUMMARY_LOG} } main