feat: export mutation testing result to Sonar (#105)

* chore(formatting): wrap braces after union and case label

* .vscode/settings.json: don't auto-add file associations

* ci(sonar): use compilation database instead of build-wrapper

* .github/workflows/static-analysis.yml: fix test report generation

* feat: export mutation testing results to Sonar

* ci: install Mull compilation driver

* ci: install Mull compilation driver

* CMakePresets.json: add mutation testing arguments as CMake list

* fix: generate paths Sonar understands

* ci(static-analysis.yml): fix access to env.GITHUB_WORKSPACE

* ci: add concurrency options and fix ccache keys

* ci: update gcovr to 5.2

* ci(static-analysis.yml): fix cache keys

* ci: use sccache for host builds

* fix(.clang-format): don't touch //NOSONAR comments

* fix(protobuf): manually apply fixes from protocolbuffers/protobuf#2968

protocolbuffers/protobuf@9ba7d1c

* fix(.clang-format): normalize line endings

* Update .github/workflows/ci.yml

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* feat(protobuf): upgrade protobuf to v21.9

* fix: protobuf Indent() changed to two spaces

* fix: Windows build; align runtime

* .gitignore: exclude Visual Studio default output dir

* Update CMakeLists.txt

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Thomas <56401213+thomas-philips@users.noreply.github.com>

.clang-format

@@ -53,13 +53,13 @@ BreakConstructorInitializers: BeforeComma
 BreakAfterJavaFieldAnnotations: false
 BreakStringLiterals: true
 ColumnLimit: 0
-CommentPragmas: "^ IWYU pragma:"
+CommentPragmas: "^ IWYU pragma:|^NOSONAR"
 QualifierAlignment: Leave
 CompactNamespaces: false
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: false
-DeriveLineEnding: true
+DeriveLineEnding: false
 DerivePointerAlignment: false
 EmptyLineAfterAccessModifier: Never
 EmptyLineBeforeAccessModifier: LogicalBlock
@@ -154,7 +154,7 @@ SpaceBeforeSquareBrackets: false
 BitFieldColonSpacing: Both
 Standard: Latest
 TabWidth: 4
-UseCRLF: false
+UseCRLF: true
 UseTab: Never
 WhitespaceSensitiveMacros:
   - STRINGIZE

.github/formatters/cobertura-to-generic-coverage.xslt

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<xsl:transform xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+    <xsl:output method="xml" indent="yes" omit-xml-declaration="yes"/>
+    <xsl:template match="/">
+        <coverage version="1">
+            <xsl:for-each select="coverage/packages/package">
+                <xsl:for-each select="classes/class">
+                    <file>
+                        <xsl:attribute name="path">
+                            <xsl:value-of select="@filename"/>
+                        </xsl:attribute>
+
+                        <xsl:for-each select="lines/line">
+                            <lineToCover>
+                                <xsl:attribute name="lineNumber">
+                                    <xsl:value-of select="@number" />
+                                </xsl:attribute>
+
+                                <xsl:attribute name="covered">
+                                    <xsl:choose>
+                                        <xsl:when test="@hits > 0">true</xsl:when>
+                                        <xsl:otherwise>false</xsl:otherwise>
+                                    </xsl:choose>
+                                </xsl:attribute>
+                            </lineToCover>
+                        </xsl:for-each>
+                    </file>
+                </xsl:for-each>
+            </xsl:for-each>
+        </coverage>
+    </xsl:template>
+</xsl:transform>

.github/formatters/mutation-report-to-generic-issue.jq

@@ -0,0 +1,52 @@
+# This jq filter transforms a mutation testing elements report to the SonarQube generic issue import format.
+
+.framework.name as $frameworkName
+| .projectRoot as $projectRoot
+| .files
+| to_entries
+| {
+    issues: map(
+        .value.mutants[] as $mutants
+        | del(.value) as $file
+        | $mutants
+        | select(.status == ("Survived", "NoCoverage"))
+        | (
+            if .replacement then
+                "The mutation operator '" + .mutatorName + "' has mutated the input to " + .replacement + " without any tests failing."
+            else
+                "The mutation operator '" + .mutatorName + "' has mutated the input without any tests failing."
+            end
+        ) as $mutation
+        | {
+            engineId: ($frameworkName // "Mutation Testing"),
+            ruleId: ("Mutant" + .status),
+            primaryLocation: {
+                message: (
+                    if .status == "NoCoverage" then
+                        "A mutant was not covered by any of the tests. " + $mutation
+                    else
+                        "A mutant survived after running the tests. " + $mutation
+                    end
+                ),
+                filePath: (
+                    if $ARGS.named["workspace"] != null then
+                        $file.key | sub("^" + $ARGS.named["workspace"] + "/"; "")
+                    elif $projectRoot then
+                        $file.key | sub("^" + $projectRoot + "/"; "")
+                    else
+                        $file.key
+                    end
+                ),
+                textRange: {
+                    startLine: .location.start.line,
+                    endLine: .location.end.line,
+                    startColumn: (.location.start.column - 1),
+                    endColumn: (.location.end.column - 1)
+                }
+            },
+            type: "CODE_SMELL",
+            severity: "MAJOR",
+            effortMinutes: 10
+        }
+    )
+}

.github/workflows/ci.yml

@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 env:
   CMAKE_BUILD_PARALLEL_LEVEL: 2
   GTEST_COLOR: 1
@@ -25,13 +29,16 @@ jobs:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
         with:
           persist-credentials: false
-      - name: Build and Test
-        run: |
-          echo "::add-matcher::.github/matchers/gcc-problem-matcher.json"
-          echo "::add-matcher::.github/matchers/msvc-problem-matcher.json"
-          cmake --preset ContinuousIntegration
-          cmake --build --preset ContinuousIntegration
-          ctest --preset ContinuousIntegration
+      - uses: hendrikmuhs/ccache-action@621a41397ed83711c72862638d9ff6e63fca3041
+        with:
+          key: ${{ github.job }}-${{ matrix.os }}
+          variant: sccache
+      - uses: lukka/run-cmake@e265d767c12ee0dc03becaa1d55155db4beb45f6
+        with:
+          configurePreset: "ContinuousIntegration"
+          buildPreset: "ContinuousIntegration"
+          testPreset: "ContinuousIntegration"
+          configurePresetCmdString: "['--preset', '$[env.CONFIGURE_PRESET_NAME]', '-D', 'CMAKE_C_COMPILER_LAUNCHER=sccache', '-D', 'CMAKE_CXX_COMPILER_LAUNCHER=sccache']"
   embedded_build:
     name: Embedded Build
     runs-on: ubuntu-latest

.github/workflows/static-analysis.yml

@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
 jobs:
   sonar:
     name: SonarCloud
@@ -26,23 +30,35 @@ jobs:
       - uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984
       - uses: BSFishy/pip-action@8f2d471d809dc20b6ada98c91910b6ae6243f318
         with:
-          packages: gcovr==5.0
-      - name: Install Sonar Scanner
+          packages: gcovr==5.2
+      - name: Install Sonar Scanner & Mull
         run: |
           wget -qN "https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip"
           unzip -qqo "sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip"
           echo "${PWD}/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> "$GITHUB_PATH"
+          wget -qN https://github.com/mull-project/mull/releases/download/0.18.0/Mull-12-0.18.0-LLVM-12.0-ubuntu-20.04.deb
+          sudo dpkg -i Mull-12-0.18.0-LLVM-12.0-ubuntu-20.04.deb
       - uses: hendrikmuhs/ccache-action@621a41397ed83711c72862638d9ff6e63fca3041
+        with:
+          key: ${{ github.job }}
+          max-size: 2G
       - name: Build & Collect Coverage
         run: |
           cmake --preset Coverage -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
           cmake --build --preset Coverage
           GTEST_OUTPUT="xml:${PWD}/testresults/" ctest --preset Coverage
           gcovr --sonarqube=coverage.xml --exclude-lines-by-pattern '.*assert\(.*\);|.*really_assert\(.*\);|.*std::abort();' --exclude-unreachable-branches --exclude-throw-branches -j 2 --exclude=.*/generated/.* --exclude=.*/examples/.* --exclude=.*/external/.* --exclude=.*/lwip/.* --exclude=.*/tracing/.* --exclude=.*/test/.*
+      - name: Build & Run Mutation Tests
+        run: |
+          cmake --preset MutationTesting -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+          cmake --build --preset MutationTesting
+          ctest --preset MutationTesting
       - name: Convert Results
         run: |
           sudo apt-get update && sudo apt-get install --no-install-recommends -y xsltproc jq
           { echo '<testExecutions version="1">'; xsltproc .github/formatters/gtest-to-generic-execution.xslt testresults/*.xml; echo '</testExecutions>'; } | tee execution.xml > /dev/null
+          jq -s 'reduce .[] as $item ({}; . * $item)' reports/mull/*.json > reports/mull/merged-mutation.json
+          jq --arg workspace "${{ env.GITHUB_WORKSPACE }}" -f .github/formatters/mutation-report-to-generic-issue.jq reports/mull/merged-mutation.json > mutation-sonar.json
           cp Build/Coverage/compile_commands.json compile_commands.json
       - name: Run Analysis
         # skip the analysis step for dependabot PRs since dependabot does not have access to secrets
@@ -66,12 +82,12 @@ jobs:
           persist-credentials: false
       - uses: seanmiddleditch/gha-setup-ninja@16b940825621068d98711680b6c3ff92201f8fc0
       - uses: hendrikmuhs/ccache-action@621a41397ed83711c72862638d9ff6e63fca3041
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6
+        with:
+          key: ${{ github.job }}
+      - uses: github/codeql-action/init@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6
         with:
           languages: cpp
       - run: |
           cmake --preset ContinuousIntegration -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
           cmake --build --preset ContinuousIntegration
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6
+      - uses: github/codeql-action/analyze@ec3cf9c605b848da5f1e41e8452719eb1ccfb9a6

.gitignore

@@ -1,8 +1,6 @@
 .vs
-Debug/
-Release/
-RelWithDebInfo/
-megalinter-reports/
 [Bb]uild*
-[Ii]nstall
+megalinter-reports/
+out/
+reports/
 CMakeSettings.json

CMakePresets.json

@@ -38,7 +38,8 @@
       "cacheVariables": {
         "CMAKE_C_COMPILER": "clang-12",
         "CMAKE_CXX_COMPILER": "clang++-12",
-        "EMIL_ENABLE_MUTATION_TESTING": "On"
+        "EMIL_ENABLE_MUTATION_TESTING": "On",
+        "EMIL_MUTATION_TESTING_RUNNER_ARGUMENTS": "--reporters;Elements;--report-dir;${sourceDir}/reports/mull"
       },
       "generator": "Ninja"
     },

cmake/emil_test_helpers.cmake

@@ -1,5 +1,6 @@
 option(EMIL_ENABLE_COVERAGE "Enable compiler flags for code coverage measurements" Off)
 option(EMIL_ENABLE_MUTATION_TESTING "Enable compiler flags for mutation testing" Off)
+set(EMIL_MUTATION_TESTING_RUNNER_ARGUMENTS "" CACHE STRING "Additional arguments for the mutation testing runner")
 
 function(emil_enable_testing)
     include(GoogleTest)
@@ -45,7 +46,7 @@ endfunction()
 
 function(emil_add_test target)
     if (EMIL_ENABLE_MUTATION_TESTING)
-        add_test(NAME ${target} COMMAND mull-runner-12 $<TARGET_FILE:${target}>)
+        add_test(NAME ${target} COMMAND mull-runner-12 ${EMIL_MUTATION_TESTING_RUNNER_ARGUMENTS} $<TARGET_FILE:${target}>)
     else()
         add_test(NAME ${target} COMMAND ${target})
     endif()

infra/stream/OutputStream.cpp

@@ -79,7 +79,7 @@ namespace infra
 
     TextOutputStream& TextOutputStream::operator<<(const char* zeroTerminatedString)
     {
-        const auto len = std::strlen(zeroTerminatedString); // NOSONAR
+        const auto len = std::strlen(zeroTerminatedString); //NOSONAR
         OutputOptionalPadding(len);
         Writer().Insert(ReinterpretCastByteRange(MakeRange(zeroTerminatedString, zeroTerminatedString + len)), ErrorPolicy());
         return *this;