ci: add C++ ASan+TSan gate on every PR; document test strategy

.github/workflows/nightly.yml

@@ -14,7 +14,10 @@ jobs:
   run-test:
     uses: ./.github/workflows/test_workflow.yml
     with:
-      configuration: '["asan"]' # Ignoring tsan for now '["asan", "tsan"]'
+      configuration: '["asan"]'
+      # C++ gtests (ASan + TSan) run on every PR via native-sanitizer-tests in ci.yml.
+      # Skip them here so the nightly focuses on Java functional tests under ASan.
+      skip_gtest: true
   report-failures:
     runs-on: ubuntu-latest
     needs: run-test

.github/workflows/test_workflow.yml

@@ -6,6 +6,11 @@ on:
       configuration:
         required: true
         type: string
+      skip_gtest:
+        description: "Skip C++ gtest execution (use when gtests run in a separate job)"
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -111,7 +116,7 @@ jobs:
 
           for attempt in $(seq 1 $MAX_ATTEMPTS); do
             mkdir -p build/logs
-            ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \
+            ./gradlew -PCI -PkeepJFRs ${{ inputs.skip_gtest == true && '-Pskip-gtest' || '' }} :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \
               | tee -a build/test-raw.log \
               | python3 -u .github/scripts/filter_gradle_log.py
             EXIT_CODE=${PIPESTATUS[0]}
@@ -399,7 +404,7 @@ jobs:
 
           for attempt in $(seq 1 $MAX_ATTEMPTS); do
             mkdir -p build/logs
-            ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \
+            ./gradlew -PCI -PkeepJFRs ${{ inputs.skip_gtest == true && '-Pskip-gtest' || '' }} :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \
               | tee -a build/test-raw.log \
               | python3 -u .github/scripts/filter_gradle_log.py
             EXIT_CODE=${PIPESTATUS[0]}

.gitlab-ci.yml

@@ -21,6 +21,7 @@ stages:
   - images
   - generate-signing-key
   - prepare
+  - sanitizer
   - build
   - stresstest
   - deploy
@@ -160,3 +161,4 @@ include:
   - local: .gitlab/benchmarks/.gitlab-ci.yml
   - local: .gitlab/reliability/.gitlab-ci.yml
   - local: .gitlab/dd-trace-integration/.gitlab-ci.yml
+  - local: .gitlab/sanitizer-tests/.gitlab-ci.yml

.gitlab/build-deploy/.gitlab-ci.yml

@@ -207,6 +207,16 @@ build-artifact:
       artifacts: true
     - job: build:arm64-musl
       artifacts: true
+    - job: gtest-asan-amd64
+      artifacts: false
+    - job: gtest-tsan-amd64
+      artifacts: false
+      optional: true
+    - job: gtest-asan-arm64
+      artifacts: false
+    - job: gtest-tsan-arm64
+      artifacts: false
+      optional: true
   when: on_success
   tags: [ "arch:amd64" ]
   image: ${BUILD_IMAGE_X64}

.gitlab/sanitizer-tests/.gitlab-ci.yml

@@ -0,0 +1,115 @@
+# C++ unit tests under ASan and TSan.
+#
+# These run on every branch push (not MR pipelines — GitHub Actions handles those).
+#
+# Strategy: use Gradle only for compile+link (buildGtest{Config}), then run
+# each binary directly from the shell. This bypasses Gradle's daemon I/O
+# which swallows child process output when fd 1/2 are not the terminal.
+
+.sanitizer_job:
+  stage: sanitizer
+  extends: .cache-config
+  needs: []
+  timeout: 30m
+  variables:
+    GRADLE_USER_HOME: .gradle
+  rules:
+    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - when: on_success
+  interruptible: true
+  before_script:
+    - apt-get update -qq
+    - apt-get install -y -qq cmake libgtest-dev libgmock-dev binutils libc6-dbg llvm
+  script:
+    - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache
+    - |
+      find ddprof-lib/build/bin/gtest -mindepth 2 -maxdepth 2 -type f -executable \
+        | grep "/${SANITIZER_LC}_" \
+        | sort \
+        | while read binary; do
+          echo ""
+          echo "=== $(basename $binary) ==="
+          "$binary"
+          rc=$?
+          if [ $rc -ne 0 ]; then
+            echo "FAILED: $(basename $binary) exited $rc"
+            exit $rc
+          fi
+        done
+  artifacts:
+    when: always
+    paths:
+      - ddprof-lib/build/bin/gtest/${SANITIZER_LC}*/
+    expire_in: 1 day
+
+gtest-asan-amd64:
+  extends: .sanitizer_job
+  allow_failure: true
+  tags: [ "arch:amd64" ]
+  image: $BUILD_IMAGE_X64
+  variables:
+    SANITIZER_CONFIG: Asan
+    SANITIZER_LC: asan
+
+gtest-tsan-amd64:
+  extends: .sanitizer_job
+  allow_failure: true
+  # docker-in-docker:amd64 = Kata Containers (kata-qemu micro VMs).
+  # Kata maps host-guest communication structures at fixed high addresses
+  # that land in TSan's shadow region regardless of LLVM version or sysctl.
+  # TSan on amd64 requires a non-Kata runner (EC2 or bare metal).
+  # Kept allow_failure so it runs and provides coverage if the environment is fixed.
+  tags: [ "docker-in-docker:amd64" ]
+  image: $BUILD_IMAGE_X64
+  variables:
+    SANITIZER_CONFIG: Tsan
+    SANITIZER_LC: tsan
+  script:
+    - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache
+    - |
+      find ddprof-lib/build/bin/gtest -mindepth 2 -maxdepth 2 -type f -executable \
+        | grep "/${SANITIZER_LC}_" | sort | while read binary; do
+          echo "=== $(basename $binary) ==="
+          GTEST_DEATH_TEST_STYLE=threadsafe "$binary"
+          rc=$?
+          [ $rc -ne 0 ] && { echo "FAILED: $(basename $binary) exited $rc"; exit $rc; }
+        done
+
+gtest-asan-arm64:
+  extends: .sanitizer_job
+  allow_failure: true
+  tags: [ "arch:arm64" ]
+  image: $BUILD_IMAGE_ARM64
+  variables:
+    SANITIZER_CONFIG: Asan
+    SANITIZER_LC: asan
+
+gtest-tsan-arm64:
+  extends: .sanitizer_job
+  allow_failure: true
+  # docker-in-docker:arm64 = EC2 VM. sysctl works directly.
+  # vm.mmap_rnd_bits=28 is sufficient — TSan's LLVM re-exec handles the rare
+  # case where a library lands in the shadow region by re-running the process
+  # via personality(ADDR_NO_RANDOMIZE).
+  # Do NOT set kernel.randomize_va_space=0: with ASLR fully off, ld-linux-aarch64.so
+  # loads at its fixed default address (0x002000000000) which is exactly TSan's
+  # 39-bit shadow start — guaranteed conflict every time.
+  tags: [ "docker-in-docker:arm64" ]
+  image: $BUILD_IMAGE_ARM64
+  variables:
+    SANITIZER_CONFIG: Tsan
+    SANITIZER_LC: tsan
+  script:
+    - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache
+    - |
+      sysctl -w vm.mmap_rnd_bits=28 2>/dev/null || true
+      find ddprof-lib/build/bin/gtest -mindepth 2 -maxdepth 2 -type f -executable \
+        | grep "/${SANITIZER_LC}_" | sort | while read binary; do
+          echo "=== $(basename $binary) ==="
+          GTEST_DEATH_TEST_STYLE=threadsafe "$binary"
+          rc=$?
+          [ $rc -ne 0 ] && { echo "FAILED: $(basename $binary) exited $rc"; exit $rc; }
+        done

build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt

@@ -187,26 +187,32 @@ object ConfigurationPresets {
                 config.compilerArgs.set(asanCompilerArgs + commonLinuxCompilerArgs(version))
 
                 val libasan = PlatformUtils.locateLibasan(compiler)
+                // Link against the sanitizer runtime that matches the compiler:
+                // - clang: locateLibasan returns libclang_rt.asan-<arch>.so, which
+                //   includes UBSan symbols; -lclang_rt.asan-<arch> satisfies -z defs
+                //   for both __asan_* and __ubsan_* and matches the runtime that
+                //   -fsanitize=address links into executables — one runtime, no conflict.
+                // - gcc: locateLibasan returns libasan.so; -lasan + -lubsan as before.
                 val asanLinkerArgs = if (libasan != null) {
-                    listOf(
-                        "-L${File(libasan).parent}",
-                        "-lasan",
-                        "-lubsan",
-                        "-fsanitize=address",
-                        "-fsanitize=undefined",
-                        "-fno-omit-frame-pointer"
-                    )
+                    val asanLibDir = File(libasan).parent
+                    val asanLibName = File(libasan).nameWithoutExtension.removePrefix("lib")
+                    val ubsanLibs = if (asanLibName.startsWith("clang_rt")) emptyList()
+                                    else listOf("-lubsan")
+                    listOf("-L$asanLibDir", "-l$asanLibName",
+                           "-Wl,-rpath,$asanLibDir") +
+                    ubsanLibs +
+                    listOf("-fsanitize=address", "-fsanitize=undefined", "-fno-omit-frame-pointer")
                 } else {
-                    emptyList()
+                    listOf("-fsanitize=address", "-fsanitize=undefined", "-fno-omit-frame-pointer")
                 }
 
                 config.linkerArgs.set(commonLinuxLinkerArgs() + asanLinkerArgs)
 
                 if (libasan != null) {
                     config.testEnvironment.apply {
                         put("LD_PRELOAD", libasan)
-                        put("ASAN_OPTIONS", "allocator_may_return_null=1:unwind_abort_on_malloc=1:use_sigaltstack=0:detect_stack_use_after_return=0:handle_segv=0:halt_on_error=0:abort_on_error=0:print_stacktrace=1:symbolize=1:log_path=/tmp/asan_%p.log:suppressions=$rootDir/gradle/sanitizers/asan.supp")
-                        put("UBSAN_OPTIONS", "halt_on_error=0:abort_on_error=0:print_stacktrace=1:log_path=/tmp/ubsan_%p.log:suppressions=$rootDir/gradle/sanitizers/ubsan.supp")
+                        put("ASAN_OPTIONS", "allocator_may_return_null=1:unwind_abort_on_malloc=1:use_sigaltstack=0:detect_stack_use_after_return=0:handle_segv=0:halt_on_error=0:abort_on_error=0:print_stacktrace=1:symbolize=1:suppressions=$rootDir/gradle/sanitizers/asan.supp")
+                        put("UBSAN_OPTIONS", "halt_on_error=0:abort_on_error=0:print_stacktrace=1:suppressions=$rootDir/gradle/sanitizers/ubsan.supp")
                         put("LSAN_OPTIONS", "detect_leaks=0")
                     }
                 }
@@ -260,7 +266,7 @@ object ConfigurationPresets {
                 if (libtsan != null) {
                     config.testEnvironment.apply {
                         put("LD_PRELOAD", libtsan)
-                        put("TSAN_OPTIONS", "suppressions=$rootDir/gradle/sanitizers/tsan.supp:log_path=/tmp/tsan_%p.log")
+                        put("TSAN_OPTIONS", "suppressions=$rootDir/gradle/sanitizers/tsan.supp")
                     }
                 }
             }

build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt

@@ -236,16 +236,16 @@ class GtestPlugin : Plugin<Project> {
         }
 
         testDir.listFiles()?.filter { it.name.endsWith(".cpp") }?.forEach { testFile ->
-            val executeTask = GtestTaskBuilder(project, extension, config)
+            val taskBundle = GtestTaskBuilder(project, extension, config)
                 .forTest(testFile)
                 .withCompiler(compiler)
                 .withIncludes(includeFiles)
                 .withSharedLibObjects(sharedLibCompileTask)
                 .onlyIfGtest(hasGtest)
                 .build()
 
-            gtestConfigTask.configure { dependsOn(executeTask) }
-            gtestAll.configure { dependsOn(executeTask) }
+            gtestConfigTask.configure { dependsOn(taskBundle) }
+            gtestAll.configure { dependsOn(taskBundle) }
             // buildGtest depends on the link task, not the run task
             buildGtestConfigTask.configure {
                 dependsOn("linkGtest${config.capitalizedName()}_${testFile.nameWithoutExtension}")

build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt

@@ -76,7 +76,7 @@ class GtestTaskBuilder(
 
     /**
      * Provide the shared library compile task whose objects are linked into
-     * every test binary.  Allows the library sources to be compiled once
+     * every test binary. Allows the 59 library sources to be compiled once
      * instead of once per test file.
      */
     fun withSharedLibObjects(task: TaskProvider<NativeCompileTask>): GtestTaskBuilder {
@@ -112,8 +112,8 @@ class GtestTaskBuilder(
             this.compiler.set(this@GtestTaskBuilder.compiler)
             this.compilerArgs.set(compilerArgs)
 
-            // When a shared library compile task is provided, library sources
-            // are compiled once there.  Only compile the test file itself here.
+            // When a shared library compile task is provided, library sources are
+            // compiled once there. Only compile the test file itself here.
             if (sharedLibCompileTask != null) {
                 sources.from(testFile)
             } else {
@@ -128,7 +128,14 @@ class GtestTaskBuilder(
     }
 
     private fun buildLinkTask(compileTask: TaskProvider<NativeCompileTask>): TaskProvider<NativeLinkExecutableTask> {
-        val linkerArgs = config.linkerArgs.get()
+        // For executables, clang's -fsanitize=address statically embeds the full
+        // ASan runtime (--whole-archive libclang_rt.asan*.a). Adding an explicit
+        // -lclang_rt.asan or -lasan on top produces a second dynamic NEEDED entry,
+        // which triggers "incompatible ASan runtimes" at startup (two __asan_init
+        // calls). Strip the explicit sanitizer -l/-L/-rpath flags here so the
+        // executable relies solely on clang's automatic static embedding.
+        val sanitizerLibPattern = Regex("^(-lasan|-lubsan|-lclang_rt\\.asan.*|-lclang_rt\\.ubsan.*|-L.*/clang.*/|-Wl,-rpath,.*/clang.*/)")
+        val linkerArgs = config.linkerArgs.get().filter { !sanitizerLibPattern.containsMatchIn(it) }
         val objDir = project.file("${project.layout.buildDirectory.get()}/obj/gtest/${config.name}/$testName")
         val binary = project.file("${project.layout.buildDirectory.get()}/bin/gtest/${config.name}_$testName/$testName")
 
@@ -191,6 +198,21 @@ class GtestTaskBuilder(
 
             inputs.files(binary)
 
+            // Gradle's default Exec task buffers child output and discards it on
+            // failure. /dev/std* bypass the logging infrastructure and stream
+            // bytes directly to fd 1/2 of the Gradle JVM so sanitizer reports
+            // are always visible in CI.
+            if (PlatformUtils.currentPlatform == Platform.LINUX) {
+                val devStdout = java.io.FileOutputStream("/dev/stdout")
+                val devStderr = java.io.FileOutputStream("/dev/stderr")
+                standardOutput = devStdout
+                errorOutput = devStderr
+                doLast {
+                    devStdout.flush(); devStdout.close()
+                    devStderr.flush(); devStderr.close()
+                }
+            }
+
             if (extension.alwaysRun.get()) {
                 outputs.upToDateWhen { false }
             }

build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt

@@ -125,7 +125,25 @@ object PlatformUtils {
         return null
     }
 
-    fun locateLibasan(compiler: String = "gcc"): String? = locateLibrary("libasan", compiler)
+    fun locateLibasan(compiler: String = "gcc"): String? {
+        if (currentPlatform != Platform.LINUX) return null
+        // For clang, prefer the architecture-specific clang_rt.asan library over
+        // GCC's libasan. Using GCC's runtime alongside clang's libclang_rt.asan
+        // (which -fsanitize=address links for executables) causes "incompatible
+        // ASan runtimes" at startup. The clang runtime also includes UBSan symbols,
+        // so no separate -lubsan is needed.
+        if (compiler.contains("clang")) {
+            val archSuffix = when (currentArchitecture) {
+                Architecture.X64 -> "x86_64"
+                Architecture.ARM64 -> "aarch64"
+                Architecture.X86 -> "i386"
+                Architecture.ARM -> "arm"
+            }
+            val clangAsan = locateLibrary("libclang_rt.asan-$archSuffix", compiler)
+            if (clangAsan != null) return clangAsan
+        }
+        return locateLibrary("libasan", compiler)
+    }
 
     fun locateLibtsan(compiler: String = "gcc"): String? = locateLibrary("libtsan", compiler)
 

ddprof-lib/src/main/cpp/gtest_crash_handler.h

@@ -118,29 +118,35 @@ void specificCrashHandler(int sig, siginfo_t *info, void *context) {
     gtestCrashHandler(sig, info, context, TestName);
 }
 
-// Install crash handler for debugging
+// Install crash handler for debugging.
+// No-op under TSan: TSan installs its own SIGSEGV/SIGBUS/SIGABRT interceptors
+// and overriding them causes TSan to crash before it can write its report.
 template<const char* TestName>
 void installGtestCrashHandler() {
+#if !defined(__SANITIZE_THREAD__)
     struct sigaction sa;
     sa.sa_flags = SA_SIGINFO;  // Get detailed info, keep handler active
     sigemptyset(&sa.sa_mask);
     sa.sa_sigaction = specificCrashHandler<TestName>;
-    
+
     // Install for various crash signals
     sigaction(SIGSEGV, &sa, nullptr);
     sigaction(SIGBUS, &sa, nullptr);
     sigaction(SIGABRT, &sa, nullptr);
     sigaction(SIGFPE, &sa, nullptr);
     sigaction(SIGILL, &sa, nullptr);
+#endif
 }
 
-// Restore default signal handlers
+// Restore default signal handlers.
 inline void restoreDefaultSignalHandlers() {
+#if !defined(__SANITIZE_THREAD__)
     signal(SIGSEGV, SIG_DFL);
     signal(SIGBUS, SIG_DFL);
     signal(SIGABRT, SIG_DFL);
     signal(SIGFPE, SIG_DFL);
     signal(SIGILL, SIG_DFL);
+#endif
 }
 
 #endif // GTEST_CRASH_HANDLER_H