sanitizers.cmake

# Source: https://github.com/StableCoder/cmake-scripts
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2018-2022 by George Cave - gcave@stablecoder.ca
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
#
# This file has been modified by Adobe.
#
# All modifications are Copyright 2020 Adobe.

include(CheckCXXSourceCompiles)

set(USE_SANITIZER
    ""
    CACHE
      STRING
      "Compile with a sanitizer. Options are: Address, Memory, MemoryWithOrigins, Undefined, Thread, Leak, 'Address;Undefined', CFI"
)

function(append value)
  foreach(variable ${ARGN})
    set(${variable}
        "${${variable}} ${value}"
        PARENT_SCOPE)
  endforeach(variable)
endfunction()

function(append_quoteless value)
  foreach(variable ${ARGN})
    set(${variable}
        ${${variable}} ${value}
        PARENT_SCOPE)
  endforeach(variable)
endfunction()

function(test_san_flags return_var flags)
  set(QUIET_BACKUP ${CMAKE_REQUIRED_QUIET})
  set(CMAKE_REQUIRED_QUIET TRUE)
  unset(${return_var} CACHE)
  set(FLAGS_BACKUP ${CMAKE_REQUIRED_FLAGS})
  set(CMAKE_REQUIRED_FLAGS "${flags}")
  check_cxx_source_compiles("int main() { return 0; }" ${return_var})
  set(CMAKE_REQUIRED_FLAGS "${FLAGS_BACKUP}")
  set(CMAKE_REQUIRED_QUIET "${QUIET_BACKUP}")
endfunction()

if(USE_SANITIZER)
  append("-fno-omit-frame-pointer" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)

  unset(SANITIZER_SELECTED_FLAGS)

  if(UNIX)

    if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
      append("-O1" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
    endif()

    if(USE_SANITIZER MATCHES "([Aa]ddress)")
      # Optional: -fno-optimize-sibling-calls -fsanitize-address-use-after-scope
      message(STATUS "Testing with Address sanitizer")
      set(SANITIZER_ADDR_FLAG "-fsanitize=address")
      test_san_flags(SANITIZER_ADDR_AVAILABLE ${SANITIZER_ADDR_FLAG})
      if(SANITIZER_ADDR_AVAILABLE)
        message(STATUS "  Building with Address sanitizer")
        append("${SANITIZER_ADDR_FLAG}" SANITIZER_SELECTED_FLAGS)

        if(AFL)
          append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER
                           CMAKE_CXX_COMPILER_LAUNCHER)
        endif()
      else()
        message(
          FATAL_ERROR
            "Address sanitizer not available for ${CMAKE_CXX_COMPILER}")
      endif()
    endif()

    if(USE_SANITIZER MATCHES "([Mm]emory([Ww]ith[Oo]rigins)?)")
      # Optional: -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2
      set(SANITIZER_MEM_FLAG "-fsanitize=memory")
      if(USE_SANITIZER MATCHES "([Mm]emory[Ww]ith[Oo]rigins)")
        message(STATUS "Testing with MemoryWithOrigins sanitizer")
        append("-fsanitize-memory-track-origins" SANITIZER_MEM_FLAG)
      else()
        message(STATUS "Testing with Memory sanitizer")
      endif()
      test_san_flags(SANITIZER_MEM_AVAILABLE ${SANITIZER_MEM_FLAG})
      if(SANITIZER_MEM_AVAILABLE)
        if(USE_SANITIZER MATCHES "([Mm]emory[Ww]ith[Oo]rigins)")
          message(STATUS "  Building with MemoryWithOrigins sanitizer")
        else()
          message(STATUS "  Building with Memory sanitizer")
        endif()
        append("${SANITIZER_MEM_FLAG}" SANITIZER_SELECTED_FLAGS)

        if(AFL)
          append_quoteless(AFL_USE_MSAN=1 CMAKE_C_COMPILER_LAUNCHER
                           CMAKE_CXX_COMPILER_LAUNCHER)
        endif()
      else()
        message(
          FATAL_ERROR
            "Memory [With Origins] sanitizer not available for ${CMAKE_CXX_COMPILER}"
        )
      endif()
    endif()

    if(USE_SANITIZER MATCHES "([Uu]ndefined)")
      message(STATUS "Testing with Undefined Behaviour sanitizer")
      set(SANITIZER_UB_FLAG "-fsanitize=undefined")
      if(EXISTS "${BLACKLIST_FILE}")
        append("-fsanitize-blacklist=${BLACKLIST_FILE}" SANITIZER_UB_FLAG)
      endif()
      test_san_flags(SANITIZER_UB_AVAILABLE ${SANITIZER_UB_FLAG})
      if(SANITIZER_UB_AVAILABLE)
        message(STATUS "  Building with Undefined Behaviour sanitizer")
        append("${SANITIZER_UB_FLAG}" SANITIZER_SELECTED_FLAGS)

        if(AFL)
          append_quoteless(AFL_USE_UBSAN=1 CMAKE_C_COMPILER_LAUNCHER
                           CMAKE_CXX_COMPILER_LAUNCHER)
        endif()
      else()
        message(
          FATAL_ERROR
            "Undefined Behaviour sanitizer not available for ${CMAKE_CXX_COMPILER}"
        )
      endif()
    endif()

    if(USE_SANITIZER MATCHES "([Tt]hread)")
      message(STATUS "Testing with Thread sanitizer")
      set(SANITIZER_THREAD_FLAG "-fsanitize=thread")
      test_san_flags(SANITIZER_THREAD_AVAILABLE ${SANITIZER_THREAD_FLAG})
      if(SANITIZER_THREAD_AVAILABLE)
        message(STATUS "  Building with Thread sanitizer")
        append("${SANITIZER_THREAD_FLAG}" SANITIZER_SELECTED_FLAGS)

        if(AFL)
          append_quoteless(AFL_USE_TSAN=1 CMAKE_C_COMPILER_LAUNCHER
                           CMAKE_CXX_COMPILER_LAUNCHER)
        endif()
      else()
        message(
          FATAL_ERROR "Thread sanitizer not available for ${CMAKE_CXX_COMPILER}"
        )
      endif()
    endif()

    if(USE_SANITIZER MATCHES "([Ll]eak)")
      message(STATUS "Testing with Leak sanitizer")
      set(SANITIZER_LEAK_FLAG "-fsanitize=leak")
      test_san_flags(SANITIZER_LEAK_AVAILABLE ${SANITIZER_LEAK_FLAG})
      if(SANITIZER_LEAK_AVAILABLE)
        message(STATUS "  Building with Leak sanitizer")
        append("${SANITIZER_LEAK_FLAG}" SANITIZER_SELECTED_FLAGS)

        if(AFL)
          append_quoteless(AFL_USE_LSAN=1 CMAKE_C_COMPILER_LAUNCHER
                           CMAKE_CXX_COMPILER_LAUNCHER)
        endif()
      else()
        message(
          FATAL_ERROR "Thread sanitizer not available for ${CMAKE_CXX_COMPILER}"
        )
      endif()
    endif()

    if(USE_SANITIZER MATCHES "([Cc][Ff][Ii])")
      message(STATUS "Testing with Control Flow Integrity(CFI) sanitizer")
      set(SANITIZER_CFI_FLAG "-fsanitize=cfi")
      test_san_flags(SANITIZER_CFI_AVAILABLE ${SANITIZER_CFI_FLAG})
      if(SANITIZER_CFI_AVAILABLE)
        message(STATUS "  Building with Control Flow Integrity(CFI) sanitizer")
        append("${SANITIZER_LEAK_FLAG}" SANITIZER_SELECTED_FLAGS)

        if(AFL)
          append_quoteless(AFL_USE_CFISAN=1 CMAKE_C_COMPILER_LAUNCHER
                           CMAKE_CXX_COMPILER_LAUNCHER)
        endif()
      else()
        message(
          FATAL_ERROR
            "Control Flow Integrity(CFI) sanitizer not available for ${CMAKE_CXX_COMPILER}"
        )
      endif()
    endif()

    message(STATUS "Sanitizer flags: ${SANITIZER_SELECTED_FLAGS}")
    test_san_flags(SANITIZER_SELECTED_COMPATIBLE ${SANITIZER_SELECTED_FLAGS})
    if(SANITIZER_SELECTED_COMPATIBLE)
      message(STATUS " Building with ${SANITIZER_SELECTED_FLAGS}")
      append("${SANITIZER_SELECTED_FLAGS}" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
    else()
      message(
        FATAL_ERROR
          " Sanitizer flags ${SANITIZER_SELECTED_FLAGS} are not compatible.")
    endif()
  elseif(MSVC)
    if(USE_SANITIZER MATCHES "([Aa]ddress)")
      message(STATUS "Building with Address sanitizer")

      # To use AddressSanitizer you need to disable incompatible options. See details here:
      # https://learn.microsoft.com/en-us/cpp/sanitizers/asan?view=msvc-170#ide-msbuild
      #
      # Please note that there are some issues with using it directly from the IDE, it may work better to
      # run the code from the command-line (and through `ctest` which enables the ASAN_SAVE_DUMPS env variable):
      # https://stackoverflow.com/questions/76781556/visual-studio-22-asan-failed-to-use-and-restart-external-symbolizer
      # https://developercommunity.visualstudio.com/t/Fail-to-use-and-restart-external-symbol/10222443?q=+Failed+to+use+and+restart+external+symbolizer
      append("/fsanitize=address" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)

      # Disable incremental linking
      add_link_options(/INCREMENTAL:NO)

      # Disable RTC flag
      foreach(flag_var
        CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
        CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO)
        string(REGEX REPLACE "/RTC[^ ]*" "" ${flag_var} "${${flag_var}}")
      endforeach(flag_var)

      # Do not use program database for debug builds (since it requires incremental linking).
      set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$<$<CONFIG:Debug,RelWithDebInfo>:Embedded>")

      if(AFL)
        append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER
                         CMAKE_CXX_COMPILER_LAUNCHER)
      endif()
    else()
      message(
        FATAL_ERROR
          "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}"
      )
    endif()
  else()
    message(FATAL_ERROR "USE_SANITIZER is not supported on this platform.")
  endif()

endif()