From fc311e347b7bfe213b3d1e57b6678b51e6152abd Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 10 Jan 2024 02:40:46 +0000 Subject: [PATCH] [Build] Support to collect the test coverage in cobertura format --- .../build-docker-sonic-vs-template.yml | 4 +- .azure-pipelines/docker-sonic-vs/Dockerfile | 8 ++++ .../test-docker-sonic-vs-template.yml | 45 +++++++++---------- azure-pipelines.yml | 1 + debian/rules | 5 ++- 5 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml index 9d1e8065fc..0ed8d7c244 100644 --- a/.azure-pipelines/build-docker-sonic-vs-template.yml +++ b/.azure-pipelines/build-docker-sonic-vs-template.yml @@ -144,9 +144,9 @@ jobs: pushd .azure-pipelines - build_args="" + build_args="--build-arg build_dir=$(pwd)" if [ '${{ parameters.asan }}' == True ]; then - build_args="--build-arg need_dbg=y" + build_args="$build_args --build-arg need_dbg=y" fi docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile index 750d136957..df08ce3205 100644 --- a/.azure-pipelines/docker-sonic-vs/Dockerfile +++ b/.azure-pipelines/docker-sonic-vs/Dockerfile @@ -2,6 +2,8 @@ FROM docker-sonic-vs ARG docker_container_name ARG need_dbg +ENV build_dir +ENV BUILD_DIR=$build_dir COPY ["debs", "/debs"] @@ -25,3 +27,9 @@ RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi RUN apt-get update RUN apt-get -y install lcov + +RUN pip install gcovr + +RUN mkdir -p $build_dir + +RUN tar -xf /tmp/gcov/gcov-source.tar -C $build_dir diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 263365d8b7..bcca22500b 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -44,6 +44,9 @@ jobs: - job: displayName: vstest timeoutInMinutes: ${{ parameters.timeout }} + variables: + DIFF_COVER_CHECK_THRESHOLD: 80 + DIFF_COVER_ENABLE: 'true' pool: sonic-common @@ -92,6 +95,7 @@ jobs: sudo apt-get install -y net-tools bridge-utils vlan sudo apt-get install -y python3-pip sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker>=4.4.1 redis==3.3.4 flaky==3.7.0 + sudo pip3 install lcov_cobertura displayName: "Install dependencies" - script: | @@ -115,6 +119,8 @@ jobs: params=" ${params} --num-ports=${{ parameters.num_ports }} " fi + cp $(Build.ArtifactStagingDirectory)/download/coverage.info ./ + all_tests=$(ls test_*.py) all_tests="${all_tests} p4rt" @@ -124,30 +130,28 @@ jobs: test_set=() # Run 20 tests as a set. + last_test=${all_tests[-1]} for test in ${all_tests}; do test_set+=("${test}") - if [ ${#test_set[@]} -ge 20 ]; then + if [ ${#test_set[@]} -ge 20 ] || [ $test == $last_test ]; then test_name=$(echo "${test_set[0]}" | cut -d "." -f 1) echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} container_count=$(docker ps -q -a | wc -l) if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then - ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory) + for container in $(docker ps | grep docker-sonic-vs | awk '{print $1}'); do + docker exec $container bash -c 'cd $BUILD_DIR; lcov -c --directory . --no-external --output-file /tmp/coverage.info' + docker cp $container:/tmp/coverage.info $container.coverage.info + done docker stop $(docker ps -q -a) docker rm $(docker ps -q -a) fi test_set=() fi done - if [ ${#test_set[@]} -gt 0 ]; then - test_name=$(echo "${test_set[0]}" | cut -d "." -f 1) - echo "${test_set[*]}" | xargs sudo py.test -v $params --force-flaky --junitxml="${test_name}_tr.xml" $params --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} - container_count=$(docker ps -q -a | wc -l) - if [ '${{ parameters.archive_gcov }}' == True ] && [ ${container_count} -gt 0 ]; then - ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory) - docker stop $(docker ps -q -a) - docker rm $(docker ps -q -a) - fi - fi + + cat *coverage.info > all_coverage.info + lcov_cobertura all_coverage.info -o coverage.xml + ls *coverage.info coverage.xml -l rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run vs tests" @@ -165,20 +169,15 @@ jobs: if [ '${{ parameters.asan }}' == True ]; then cp -vr tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/ fi - - if [ '${{ parameters.archive_gcov }}' == True ]; then - sudo apt-get install -y lcov - cd $(Build.ArtifactStagingDirectory)/gcov_tmp/ - tar -zcvf sonic-gcov.tar.gz sonic-gcov/ - rm -rf sonic-gcov - fi displayName: "Collect logs" condition: always() - - publish: $(Build.ArtifactStagingDirectory)/gcov_tmp - artifact: ${{ parameters.gcov_artifact_name }} - displayName: "Publish gcov output" - condition: and(succeeded(), eq('${{ parameters.archive_gcov }}', true)) + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml' + reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/' + displayName: 'Publish test coverage' - publish: $(Build.ArtifactStagingDirectory)/ artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 362a106225..ca2dd8e63d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -137,6 +137,7 @@ stages: asan: true - stage: Gcov + condition: false dependsOn: Test condition: in(dependencies.Test.result, 'Succeeded', 'SucceededWithIssues') jobs: diff --git a/debian/rules b/debian/rules index 42e82b2f30..2493d3aa92 100755 --- a/debian/rules +++ b/debian/rules @@ -43,7 +43,10 @@ override_dh_auto_install: dh_auto_install --destdir=debian/swss ifeq ($(ENABLE_GCOV), y) mkdir -p debian/swss/tmp/gcov - sh ./tests/gcov_support.sh collect swss + lcov -c --directory . --no-external --output-file coverage.info + tar -cf my_archive.tar $$(shell ) + find ./ -type f -regex '.*\.\(h\|ccp\|gcno|info\)' | tar -cf debian/swss/tmp/gcov/gcov-source.tar -T - + #sh ./tests/gcov_support.sh collect swss endif override_dh_strip: