Also run Codecov coverage recording on macOS

Codecov will merge multiple reports as documented at https://docs.codecov.com/docs/merging-reports.
tautschnig · Jun 19, 2024 · 22a65db · 22a65db
1 parent edcf648
commit 22a65db
Show file tree

Hide file tree

Showing 2 changed files with 144 additions and 2 deletions.
diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml
@@ -179,3 +179,145 @@ jobs:
           files: build/html/coverage.info
           fail_ci_if_error: true
           verbose: true
+
+  # This job takes approximately 36 to 85 minutes
+  macOS-cache-build:
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Fetch dependencies
+        run: |
+          # maven is already installed and upgrading to a newer version yields
+          # symlink conflicts as previously reported in https://github.com/actions/runner-images/issues/4020
+          brew install cmake flex bison llvm lcov ccache z3
+      - name: Prepare ccache
+        uses: actions/cache@v4
+        with:
+          save-always: true
+          path: .ccache
+          key: ${{ runner.os }}-Coverage-${{ github.ref }}-${{ github.sha }}-PR
+          restore-keys: |
+            ${{ runner.os }}-Coverage-${{ github.ref }}
+            ${{ runner.os }}-Coverage
+      - name: ccache environment
+        run: |
+          echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
+          echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
+      - name: Configure CMake CBMC build with coverage instrumentation parameters
+        run: cmake -S . -Bbuild -Denable_coverage=1 -Dparallel_tests=4 -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++
+      - name: Zero ccache stats and limit in size
+        run: ccache -z --max-size=7G
+      - name: Execute CMake CBMC build
+        run: cmake --build build -- -j4
+      - name: Print ccache stats
+        run: ccache -s
+
+  # This job takes approximately 36 to 85 minutes
+  macOS-non-JBMC:
+    needs: macOS-cache-build
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Fetch dependencies
+        run: |
+          # maven is already installed and upgrading to a newer version yields
+          # symlink conflicts as previously reported in https://github.com/actions/runner-images/issues/4020
+          brew install cmake flex bison llvm lcov ccache z3
+      - name: Confirm z3 solver is available and log the version installed
+        run: z3 --version
+      - name: Download cvc5 binary and make sure it can be deployed
+        run: |
+          curl -L https://github.com/cvc5/cvc5/releases/download/cvc5-${{env.cvc5-version}}/cvc5-macOS --output cvc5
+          chmod u+x cvc5
+          mv cvc5 /usr/local/bin
+          cvc5 --version
+      - name: Prepare ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: .ccache
+          key: ${{ runner.os }}-Coverage-${{ github.ref }}-${{ github.sha }}-PR
+          restore-keys: |
+            ${{ runner.os }}-Coverage-${{ github.ref }}
+            ${{ runner.os }}-Coverage
+      - name: ccache environment
+        run: |
+          echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
+          echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
+      - name: Configure CMake CBMC build with coverage instrumentation parameters
+        run: cmake -S . -Bbuild -Denable_coverage=1 -Dparallel_tests=4 -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCOVERAGE_CTEST_EXTRA_FLAGS="-E;^jbmc" -DLCOV_CAPTURE_FLAGS="--ignore-errors;inconsistent"
+      - name: Zero ccache stats and limit in size
+        run: ccache -z --max-size=7G
+      - name: Execute CMake CBMC build
+        run: cmake --build build -- -j4
+      - name: Print ccache stats
+        run: ccache -s
+      - name: Run CTest and collect coverage statistics
+        run: |
+          echo "lcov_excl_line = UNREACHABLE" > ~/.lcovrc
+          echo "geninfo_external = 0" >> ~/.lcovrc
+          cmake --build build --target coverage -- -j4
+      - name: Upload coverage statistics to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: build/html/coverage.info
+          fail_ci_if_error: true
+          verbose: true
+
+  # This job takes approximately 36 to 85 minutes
+  macOS-JBMC:
+    needs: macOS-cache-build
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Fetch dependencies
+        run: |
+          # maven is already installed and upgrading to a newer version yields
+          # symlink conflicts as previously reported in https://github.com/actions/runner-images/issues/4020
+          brew install cmake flex bison llvm lcov ccache z3
+      - name: Confirm z3 solver is available and log the version installed
+        run: z3 --version
+      - name: Download cvc5 binary and make sure it can be deployed
+        run: |
+          curl -L https://github.com/cvc5/cvc5/releases/download/cvc5-${{env.cvc5-version}}/cvc5-macOS --output cvc5
+          chmod u+x cvc5
+          mv cvc5 /usr/local/bin
+          cvc5 --version
+      - name: Prepare ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: .ccache
+          key: ${{ runner.os }}-Coverage-${{ github.ref }}-${{ github.sha }}-PR
+          restore-keys: |
+            ${{ runner.os }}-Coverage-${{ github.ref }}
+            ${{ runner.os }}-Coverage
+      - name: ccache environment
+        run: |
+          echo "CCACHE_BASEDIR=$PWD" >> $GITHUB_ENV
+          echo "CCACHE_DIR=$PWD/.ccache" >> $GITHUB_ENV
+      - name: Configure CMake CBMC build with coverage instrumentation parameters
+        run: cmake -S . -Bbuild -Denable_coverage=1 -Dparallel_tests=4 -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCOVERAGE_CTEST_EXTRA_FLAGS="-R;^jbmc" -DLCOV_CAPTURE_FLAGS="--ignore-errors;inconsistent"
+      - name: Zero ccache stats and limit in size
+        run: ccache -z --max-size=7G
+      - name: Execute CMake CBMC build
+        run: cmake --build build -- -j4
+      - name: Print ccache stats
+        run: ccache -s
+      - name: Run CTest and collect coverage statistics
+        run: |
+          echo "lcov_excl_line = UNREACHABLE" > ~/.lcovrc
+          echo "geninfo_external = 0" >> ~/.lcovrc
+          cmake --build build --target coverage -- -j4
+      - name: Upload coverage statistics to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: build/html/coverage.info
+          fail_ci_if_error: true
+          verbose: true
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -156,8 +156,8 @@ if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR
         add_custom_target(coverage
             COMMAND ${CMAKE_COMMAND} -E make_directory ${CODE_COVERAGE_OUTPUT_DIR}
             COMMAND ctest -V -L CORE -j${parallel_tests} ${COVERAGE_CTEST_EXTRA_FLAGS}
-            COMMAND ${CODE_COVERAGE_LCOV} ${LCOV_FLAGS} --capture --directory ${CMAKE_BINARY_DIR} --output-file ${CODE_COVERAGE_INFO_FILE}
-            COMMAND ${CODE_COVERAGE_LCOV} ${LCOV_FLAGS} --remove ${CODE_COVERAGE_INFO_FILE} '/usr/*' --output-file ${CODE_COVERAGE_INFO_FILE}
+            COMMAND ${CODE_COVERAGE_LCOV} --capture ${LCOV_CAPTURE_FLAGS} --directory ${CMAKE_BINARY_DIR} --output-file ${CODE_COVERAGE_INFO_FILE}
+            COMMAND ${CODE_COVERAGE_LCOV} --remove ${LCOV_REMOVE_FLAGS} ${CODE_COVERAGE_INFO_FILE} '/usr/*' --output-file ${CODE_COVERAGE_INFO_FILE}
             COMMAND ${CODE_COVERAGE_GENHTML} ${CODE_COVERAGE_INFO_FILE} --output-directory ${CODE_COVERAGE_OUTPUT_DIR}
             DEPENDS
               "$<TARGET_FILE:java-unit>"