iox-#692 Add Thread Sanitizer

Signed-off-by: Ibrahim Kuru <ibrahim.kuru@apex.ai>

.github/workflows/build-test.yml

@@ -82,8 +82,8 @@ jobs:
       - uses: actions/checkout@v3
       - run: ./tools/ci/run-integration-test.sh
 
-  build-test-ubuntu-with-sanitizers-gcc-latest:
-    # prevent stuck jobs consuming runners for 6 hours
+  build-test-ubuntu-with-address-sanitizer-gcc-latest:
+    # prevent stuck jobs consuming runners for 1 hour
     timeout-minutes: 60
     runs-on: ubuntu-20.04
     needs: pre-flight-check
@@ -93,10 +93,11 @@ jobs:
           version: 11
           platform: x64
       - uses: actions/checkout@v3
-      - run: ./tools/ci/build-test-ubuntu-with-sanitizers.sh gcc
+      - name: Run Address Sanitizer
+        run: ./tools/ci/build-test-ubuntu-with-sanitizers.sh gcc asan
 
-  build-test-ubuntu-with-sanitizers-clang-latest:
-    # prevent stuck jobs consuming runners for 6 hours
+  build-test-ubuntu-with-address-sanitizer-clang-latest:
+    # prevent stuck jobs consuming runners for 1 hour
     timeout-minutes: 60
     runs-on: ubuntu-latest
     needs: pre-flight-check
@@ -105,17 +106,32 @@ jobs:
         uses: actions/checkout@v3
       - name: Install iceoryx dependencies and clang-tidy
         uses: ./.github/actions/install-iceoryx-deps-and-clang
-      - name: build with sanitizers
-        run: ./tools/ci/build-test-ubuntu-with-sanitizers.sh clang
+      - name: Run Address Sanitizer
+        run: ./tools/ci/build-test-ubuntu-with-sanitizers.sh clang asan
 
-  build-test-macos-with-sanitizers:
-    # prevent stuck jobs consuming runners for 6 hours
+  build-test-ubuntu-with-thread-sanitizer-clang-latest:
+    # prevent stuck jobs consuming runners for 3 hours
+    timeout-minutes: 180
+    runs-on: ubuntu-latest
+    needs: pre-flight-check
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install iceoryx dependencies and clang-tidy
+        uses: ./.github/actions/install-iceoryx-deps-and-clang
+      - name: Run Thread Sanitizer
+        run: ./tools/ci/build-test-ubuntu-with-sanitizers.sh clang tsan
+
+  build-test-macos-with-address-sanitizer:
+    # prevent stuck jobs consuming runners for 1 hour
     timeout-minutes: 60
-    runs-on: macos-latest
+    runs-on: macos-12
     needs: pre-flight-check
     steps:
       - uses: actions/checkout@v3
-      - run: ./tools/ci/build-test-macos-with-sanitizers.sh
+      - name: Run Address Sanitizer
+        run: ./tools/ci/build-test-macos-with-sanitizers.sh asan
 
   # gcc 5.4 is compiler used in QNX 7.0
   build-test-ubuntu-with-gcc54:

doc/website/release-notes/iceoryx-unreleased.md

@@ -107,6 +107,7 @@
 
 - Remove hash from the branch names [\#1530](https://github.com/eclipse-iceoryx/iceoryx/issues/1530)
 - Automate check for test cases to have UUIDs [\#1540](https://github.com/eclipse-iceoryx/iceoryx/issues/1540)
+- Add Thread Sanitizer to build and test workflow [\#692](https://github.com/eclipse-iceoryx/iceoryx/issues/692)
 
 **New API features:**
 

iceoryx_hoofs/cmake/IceoryxPlatform.cmake

@@ -49,6 +49,25 @@ function(iox_create_asan_runtime_blacklist BLACKLIST_FILE_PATH)
     endif()
 endfunction()
 
+function(iox_create_tsan_runtime_blacklist BLACKLIST_FILE_PATH)
+    # (https://github.com/google/sanitizers/wiki/ThreadSanitizerSuppressions)
+    # The suppression types are:
+    # race              suppresses data races and use-after-free reports
+    # race_top          same as race, but matched only against the top stack frame
+    # thread            suppresses reports related to threads (leaks)
+    # mutex             suppresses reports related to mutexes (destruction of a locked mutex)
+    # signal            suppresses reports related to signal handlers (handler calls malloc())
+    # deadlock          suppresses lock inversion reports
+    # called_from_lib   suppresses all interceptors in a particular library
+    if(NOT EXISTS ${BLACKLIST_FILE_PATH})
+        file(WRITE  ${BLACKLIST_FILE_PATH} "# This file is auto-generated from iceoryx_hoofs/cmake/IceoryxPlatform.cmake\n")
+        file(APPEND ${BLACKLIST_FILE_PATH} "#mutex:*\n")
+        file(APPEND ${BLACKLIST_FILE_PATH} "#race:*\n")
+        file(APPEND ${BLACKLIST_FILE_PATH} "#deadlock:*\n")
+        file(APPEND ${BLACKLIST_FILE_PATH} "# End of file\n")
+    endif()
+endfunction()
+
 function(iox_create_lsan_runtime_blacklist BLACKLIST_FILE_PATH)
     # Suppress known memory leaks (https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer)
     # Below function/files contains memory leaks!
@@ -68,7 +87,11 @@ function(iox_create_lsan_runtime_blacklist BLACKLIST_FILE_PATH)
     endif()
 endfunction()
 
-if(SANITIZE)
+if(ADDRESS_SANITIZER OR THREAD_SANITIZER)
+    if(ADDRESS_SANITIZER AND THREAD_SANITIZER)
+        message( FATAL_ERROR "You can not run asan (address sanitizer) and tsan (thread sanitizer) together. Deselect one of them!" )
+    endif()
+
     if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
         set(ICEORYX_SANITIZER_BLACKLIST_FILE ${CMAKE_BINARY_DIR}/sanitizer_blacklist/sanitizer_compile_time.txt)
         iox_create_asan_compile_time_blacklist(${ICEORYX_SANITIZER_BLACKLIST_FILE})
@@ -81,6 +104,7 @@ if(SANITIZE)
 
     iox_create_asan_runtime_blacklist(${CMAKE_BINARY_DIR}/sanitizer_blacklist/asan_runtime.txt)
     iox_create_lsan_runtime_blacklist(${CMAKE_BINARY_DIR}/sanitizer_blacklist/lsan_runtime.txt)
+    iox_create_tsan_runtime_blacklist(${CMAKE_BINARY_DIR}/sanitizer_blacklist/tsan_runtime.txt)
 
     if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
         set(ICEORYX_SANITIZER_COMMON_FLAGS -fno-omit-frame-pointer -fno-optimize-sibling-calls)
@@ -96,14 +120,22 @@ if(SANITIZE)
         # -fno-sanitize-recover=... print a verbose error report and exit the program
         set(ICEORYX_UB_SANITIZER_FLAGS -fsanitize=undefined -fno-sanitize-recover=undefined)
 
-        # Combine different sanitizer flags to define overall sanitization
-        set(ICEORYX_SANITIZER_FLAGS ${ICEORYX_SANITIZER_COMMON_FLAGS} ${ICEORYX_ADDRESS_SANITIZER_FLAGS} ${ICEORYX_UB_SANITIZER_FLAGS} CACHE INTERNAL "")
+        # ThreadSanitizer
+        set(ICEORYX_THREAD_SANITIZER_FLAGS -fsanitize=thread)
 
+        # Combine different sanitizer flags to define overall sanitization
+        if(ADDRESS_SANITIZER)
+            set(ICEORYX_SANITIZER_FLAGS ${ICEORYX_SANITIZER_COMMON_FLAGS} ${ICEORYX_ADDRESS_SANITIZER_FLAGS} ${ICEORYX_UB_SANITIZER_FLAGS} CACHE INTERNAL "")
+        elseif(THREAD_SANITIZER)
+            set(ICEORYX_SANITIZER_FLAGS ${ICEORYX_SANITIZER_COMMON_FLAGS} ${ICEORYX_THREAD_SANITIZER_FLAGS} CACHE INTERNAL "")
+        endif()
+
         # unset local variables , to avoid polluting global space
         unset(ICEORYX_SANITIZER_BLACKLIST)
         unset(ICEORYX_SANITIZER_COMMON_FLAGS)
         unset(ICEORYX_ADDRESS_SANITIZER_FLAGS)
         unset(ICEORYX_UB_SANITIZER_FLAGS)
+        unset(ICEORYX_THREAD_SANITIZER_FLAGS)
     else()
         message( FATAL_ERROR "You need to run sanitize with gcc/clang compiler." )
     endif()

iceoryx_meta/build_options.cmake

@@ -31,7 +31,8 @@ option(INTROSPECTION "Builds the introspection client which requires the ncurses
 option(ONE_TO_MANY_ONLY "Restricts communication to 1:n pattern" OFF)
 set(IOX_PLATFORM_PATH "" CACHE PATH "Overrides integrated platform detection and uses provided custom path")
 option(ROUDI_ENVIRONMENT "Build RouDi Environment for testing, is enabled when building tests" OFF)
-option(SANITIZE "Build with sanitizers" OFF)
+option(ADDRESS_SANITIZER "Build with address sanitizer" OFF)
+option(THREAD_SANITIZER "Build with thread sanitizer" OFF)
 option(TEST_WITH_ADDITIONAL_USER "Build Test with additional user accounts for testing access control" OFF)
 option(TOML_CONFIG "TOML support for RouDi with dynamic configuration" ON)
 
@@ -86,7 +87,8 @@ function(show_config_options)
   message("          ONE_TO_MANY_ONLY ....................: " ${ONE_TO_MANY_ONLY})
   message("          IOX_PLATFORM_PATH....................: " ${IOX_PLATFORM_PATH})
   message("          ROUDI_ENVIRONMENT....................: " ${ROUDI_ENVIRONMENT} ${ROUDI_ENV_HINT})
-  message("          SANITIZE.............................: " ${SANITIZE})
+  message("          ADDRESS_SANITIZER....................: " ${ADDRESS_SANITIZER})
+  message("          THREAD_SANITIZER.....................: " ${THREAD_SANITIZER})
   message("          TEST_WITH_ADDITIONAL_USER ...........: " ${TEST_WITH_ADDITIONAL_USER})
   message("          TOML_CONFIG..........................: " ${TOML_CONFIG})
 endfunction()

tools/ci/build-test-macos-with-sanitizers.sh

@@ -19,10 +19,17 @@
 
 set -e
 
+SANITIZER=${1:-asan}
+
 msg() {
     printf "\033[1;32m%s: %s\033[0m\n" ${FUNCNAME[1]} "$1"
 }
 
+if [ "$SANITIZER" != "asan" && "$SANITIZER" != "tsan" ]; then
+    msg "Invalid sanitizer."
+    exit 1
+fi
+
 WORKSPACE=$(git rev-parse --show-toplevel)
 cd ${WORKSPACE}
 
@@ -37,9 +44,15 @@ cd "${WORKSPACE}"
 msg "building sources"
 export LDFLAGS="-L/usr/local/opt/ncurses/lib"
 export CFLAGS="-I/usr/local/opt/ncurses/include"
-./tools/iceoryx_build_test.sh build-strict build-all sanitize
+./tools/iceoryx_build_test.sh build-strict build-all $SANITIZER
 
 msg "running tests (excluding timing_tests)"
 cd ./build
-tools/run_tests.sh asan-only
+if [ "$SANITIZER" == "asan" ]; then
+    tools/run_tests.sh asan-only
+elif [ "$SANITIZER" == "tsan" ]; then
+    tools/run_tests.sh tsan-only
+else
+    tools/run_tests.sh all
+fi
 cd -

tools/ci/build-test-ubuntu-with-sanitizers.sh

@@ -20,6 +20,7 @@
 set -e
 
 COMPILER=${1:-gcc}
+SANITIZER=${2:-asan}
 
 msg() {
     printf "\033[1;32m%s: %s\033[0m\n" ${FUNCNAME[1]} "$1"
@@ -34,17 +35,25 @@ sudo apt-get update && sudo apt-get install -y libacl1-dev libncurses5-dev
 msg "creating local test users and groups for testing access control"
 sudo ./tools/scripts/add_test_users.sh
 
+if [ "$SANITIZER" != "asan" ] && [ "$SANITIZER" != "tsan" ]; then
+    msg "Invalid sanitizer."
+    exit 1
+fi
 
 msg "building sources"
 if [ "$COMPILER" == "gcc" ]; then
-    ./tools/iceoryx_build_test.sh clean build-strict build-shared build-all debug sanitize test-add-user out-of-tree
+    ./tools/iceoryx_build_test.sh clean build-strict build-shared build-all debug $SANITIZER test-add-user out-of-tree
 fi
 
 if [ "$COMPILER" == "clang" ]; then
-    ./tools/iceoryx_build_test.sh clean build-strict build-shared build-all clang debug sanitize test-add-user out-of-tree
+    ./tools/iceoryx_build_test.sh clean build-strict build-shared build-all clang debug $SANITIZER test-add-user out-of-tree
 fi
 
 msg "running all tests"
 cd ./build
-./tools/run_tests.sh all
+if [ "$SANITIZER" == "tsan" ]; then
+    ./tools/run_tests.sh all continue-on-error
+else
+    ./tools/run_tests.sh all
+fi
 cd -

tools/iceoryx_build_test.sh

@@ -41,7 +41,8 @@ TEST_SCOPE="all" #possible values for test scope: 'all', 'unit', 'integration'
 RUN_TEST=false
 BINDING_C_FLAG="ON"
 ONE_TO_MANY_ONLY_FLAG="OFF"
-SANITIZE_FLAG="OFF"
+ADDRESS_SANITIZER_FLAG="OFF"
+THREAD_SANITIZER_FLAG="OFF"
 ROUDI_ENV_FLAG="OFF"
 TEST_ADD_USER="OFF"
 OUT_OF_TREE_FLAG="OFF"
@@ -161,10 +162,17 @@ while (( "$#" )); do
         TOML_FLAG="OFF"
         shift 1
         ;;
-    "sanitize")
-        echo " [i] Build with sanitizers"
+    "asan")
+        echo " [i] Build with address sanitizer"
         BUILD_TYPE="Debug"
-        SANITIZE_FLAG="ON"
+        ADDRESS_SANITIZER_FLAG="ON"
+        BUILD_SHARED="OFF"
+        shift 1
+        ;;
+    "tsan")
+        echo " [i] Build with thread sanitizer"
+        BUILD_TYPE="Debug"
+        THREAD_SANITIZER_FLAG="ON"
         BUILD_SHARED="OFF"
         shift 1
         ;;
@@ -211,7 +219,8 @@ while (( "$#" )); do
         echo "    one-to-many-only      Restrict to 1:n communication only"
         echo "    out-of-tree           Out-of-tree build for CI"
         echo "    package               Create a debian package from clean build in build_package"
-        echo "    sanitize              Build with sanitizers"
+        echo "    asan                  Build with address sanitizer"
+        echo "    tsan                  Build with thread sanitizer"
         echo "    release               Build with -O3"
         echo "    relwithdebinfo        Build with -O2 -DNDEBUG"
         echo "    test                  Build and run all tests in all iceoryx components"
@@ -284,7 +293,8 @@ if [ "$NO_BUILD" == false ]; then
           -DBINDING_C=$BINDING_C_FLAG \
           -DONE_TO_MANY_ONLY=$ONE_TO_MANY_ONLY_FLAG \
           -DBUILD_SHARED_LIBS=$BUILD_SHARED \
-          -DSANITIZE=$SANITIZE_FLAG \
+          -DADDRESS_SANITIZER=$ADDRESS_SANITIZER_FLAG \
+          -DTHREAD_SANITIZER=$THREAD_SANITIZER_FLAG \
           -DTEST_WITH_ADDITIONAL_USER=$TEST_ADD_USER $TOOLCHAIN_FILE \
           -DCMAKE_CXX_FLAGS=$CMAKE_CXX_FLAGS \
           "$WORKSPACE"/iceoryx_meta

tools/run_tests.sh

@@ -23,6 +23,7 @@ BASE_DIR=$PWD
 TEST_SCOPE="all"
 CONTINUE_ON_ERROR=false
 ASAN_ONLY=false
+test_failed=0
 
 set_sanitizer_options() {
     # This script runs from build folder
@@ -45,10 +46,13 @@ set_sanitizer_options() {
     export LSAN_OPTIONS
     UBSAN_OPTIONS=print_stacktrace=1
     export UBSAN_OPTIONS
+    TSAN_OPTIONS=suppressions=$BASE_DIR/sanitizer_blacklist/tsan_runtime.txt
+    export TSAN_OPTIONS
 
     echo "ASAN_OPTIONS : $ASAN_OPTIONS"
     echo "LSAN_OPTIONS : $LSAN_OPTIONS"
     echo "UBSAN_OPTIONS : $UBSAN_OPTIONS"
+    echo "TSAN_OPTIONS : $TSAN_OPTIONS"
 
     if [[ ! -f $(which llvm-symbolizer) ]]
     then
@@ -65,6 +69,10 @@ for arg in "$@"; do
         ASAN_ONLY=true
         TEST_SCOPE="no_timing_test"
         ;;
+    "tsan-only")
+        CONTINUE_ON_ERROR=true
+        TEST_SCOPE="no_timing_test"
+        ;;
     "continue-on-error")
         CONTINUE_ON_ERROR=true
         ;;
@@ -82,6 +90,7 @@ for arg in "$@"; do
         echo "      only-timing-tests           Runs only timing tests"
         echo "      continue-on-error           Continue execution upon error"
         echo "      asan-only                   Execute Adress-Sanitizer only"
+        echo "      tsan-only                   Execute Thread-Sanitizer only"
         echo ""
         exit 1
         ;;
@@ -111,6 +120,17 @@ fi
 
 set_sanitizer_options
 
+make_c() {
+    make $1 || test_failed=1
+    if [ "$test_failed" == "1" ]; then
+        echo "------->>>>> Test: $1 failed!!!!"
+        if [ "$CONTINUE_ON_ERROR" != "true" ]; then
+            echo "Exiting immediately (CONTINUE_ON_ERROR=false)"
+            exit 1
+        fi
+    fi
+}
+
 execute_test() {
     local test_scope=$1
 
@@ -119,30 +139,32 @@ execute_test() {
 
     case $test_scope in
     "all")
-        make all_tests
-        make timing_module_tests
-        make timing_integration_tests
+        make_c all_tests
+        make_c timing_module_tests
+        make_c timing_integration_tests
         ;;
     "no_timing_test")
-        make all_tests
+        make_c all_tests
         ;;
     "unit")
-        make module_tests
+        make_c module_tests
         ;;
     "unit-timing")
-        make timing_module_tests
+        make_c timing_module_tests
         ;;
     "integration")
-        make integration_tests
+        make_c integration_tests
         ;;
     "timingtest")
-        make timing_module_tests
-        make timing_integration_tests
+        make_c timing_module_tests
+        make_c timing_integration_tests
         ;;
     *)
         echo "Wrong scope $test_scope!"
         ;;
     esac
+
+    return $((test_failed))
 }
 
 execute_test "$TEST_SCOPE"