diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index a55f01fd52789b..71590be60f557e 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -154,6 +154,15 @@ message(STATUS "build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}" option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for not" OFF) message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") +option(ENABLE_EXPR_ASAN "Build Exprs target with ASAN/UBSAN instrumentation" OFF) +message(STATUS "enable expr asan: ${ENABLE_EXPR_ASAN}") + +set(DORIS_BE_ASAN_SOURCES "" CACHE STRING "Comma-separated list of BE source files to compile with ASAN/UBSAN instrumentation") +if (DEFINED ENV{DORIS_BE_ASAN_SOURCES} AND NOT DORIS_BE_ASAN_SOURCES) + set(DORIS_BE_ASAN_SOURCES "$ENV{DORIS_BE_ASAN_SOURCES}" CACHE STRING "" FORCE) +endif() +message(STATUS "be asan sources: ${DORIS_BE_ASAN_SOURCES}") + option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") @@ -453,6 +462,122 @@ set(CXX_FLAGS_ASAN "-O0 -fsanitize=address -fsanitize=undefined -fno-sanitize=fl if (COMPILER_CLANG) set(CXX_FLAGS_ASAN "${CXX_FLAGS_ASAN} -fsanitize-ignorelist=${UBSAN_IGNORELIST}") endif() +set(DORIS_ASAN_COMPILE_OPTIONS + -O0 + -fsanitize=address + -fsanitize=undefined + -fno-sanitize=float-cast-overflow +) +set(DORIS_ASAN_LINK_OPTIONS + -fsanitize=address + -fsanitize=undefined +) +if (COMPILER_CLANG) + list(APPEND DORIS_ASAN_COMPILE_OPTIONS -fsanitize-ignorelist=${UBSAN_IGNORELIST}) +elseif (COMPILER_GCC) + list(APPEND DORIS_ASAN_LINK_OPTIONS -static-libasan) +endif() + +function(enable_doris_asan target) + target_compile_options(${target} PRIVATE ${DORIS_ASAN_COMPILE_OPTIONS}) + target_compile_definitions(${target} PRIVATE + ADDRESS_SANITIZER + UNDEFINED_BEHAVIOR_SANITIZER + ) +endfunction() + +function(link_doris_asan_runtime target) + target_link_options(${target} PRIVATE ${DORIS_ASAN_LINK_OPTIONS}) +endfunction() + +set(DORIS_BE_ASAN_SOURCE_LIST) +if (DORIS_BE_ASAN_SOURCES) + string(REPLACE "," ";" DORIS_BE_ASAN_SOURCE_CANDIDATES "${DORIS_BE_ASAN_SOURCES}") + foreach(source_candidate IN LISTS DORIS_BE_ASAN_SOURCE_CANDIDATES) + string(STRIP "${source_candidate}" source_candidate) + if (source_candidate STREQUAL "") + continue() + endif() + + set(source_candidate_paths) + if (IS_ABSOLUTE "${source_candidate}") + list(APPEND source_candidate_paths "${source_candidate}") + else() + list(APPEND source_candidate_paths + "${BASE_DIR}/${source_candidate}" + "${BASE_DIR}/../${source_candidate}" + ) + endif() + + set(resolved_source_candidate) + foreach(source_candidate_path IN LISTS source_candidate_paths) + if (EXISTS "${source_candidate_path}") + get_filename_component(resolved_source_candidate "${source_candidate_path}" REALPATH) + break() + endif() + endforeach() + + if (NOT resolved_source_candidate) + message(FATAL_ERROR "DORIS_BE_ASAN_SOURCES entry '${source_candidate}' does not exist") + endif() + + string(FIND "${resolved_source_candidate}" "${BASE_DIR}/src/" be_src_pos) + if (NOT be_src_pos EQUAL 0) + message(FATAL_ERROR "DORIS_BE_ASAN_SOURCES entry '${source_candidate}' must be under ${BASE_DIR}/src") + endif() + + list(APPEND DORIS_BE_ASAN_SOURCE_LIST "${resolved_source_candidate}") + endforeach() + + list(REMOVE_DUPLICATES DORIS_BE_ASAN_SOURCE_LIST) +endif() + +function(enable_doris_asan_for_sources target) + if (NOT DORIS_BE_ASAN_SOURCE_LIST) + return() + endif() + + get_target_property(target_sources ${target} SOURCES) + if (NOT target_sources) + return() + endif() + + get_target_property(target_source_dir ${target} SOURCE_DIR) + foreach(target_source IN LISTS target_sources) + get_filename_component(target_source_abs "${target_source}" ABSOLUTE BASE_DIR "${target_source_dir}") + list(FIND DORIS_BE_ASAN_SOURCE_LIST "${target_source_abs}" source_index) + if (source_index EQUAL -1) + continue() + endif() + + set_property(SOURCE "${target_source_abs}" TARGET_DIRECTORY ${target} APPEND PROPERTY + COMPILE_OPTIONS ${DORIS_ASAN_COMPILE_OPTIONS}) + set_property(SOURCE "${target_source_abs}" TARGET_DIRECTORY ${target} APPEND PROPERTY + COMPILE_DEFINITIONS ADDRESS_SANITIZER UNDEFINED_BEHAVIOR_SANITIZER) + set_property(SOURCE "${target_source_abs}" TARGET_DIRECTORY ${target} PROPERTY + SKIP_PRECOMPILE_HEADERS ON) + set_property(GLOBAL APPEND PROPERTY DORIS_BE_ASAN_MATCHED_SOURCES "${target_source_abs}") + message(STATUS "Enable BE source ASAN for target ${target}: ${target_source_abs}") + endforeach() +endfunction() + +function(validate_doris_asan_sources) + if (NOT DORIS_BE_ASAN_SOURCE_LIST) + return() + endif() + + get_property(matched_sources GLOBAL PROPERTY DORIS_BE_ASAN_MATCHED_SOURCES) + if (matched_sources) + list(REMOVE_DUPLICATES matched_sources) + endif() + + foreach(requested_source IN LISTS DORIS_BE_ASAN_SOURCE_LIST) + list(FIND matched_sources "${requested_source}" matched_index) + if (matched_index EQUAL -1) + message(FATAL_ERROR "Requested BE ASAN source '${requested_source}' is not compiled by any registered BE target") + endif() + endforeach() +endfunction() set(CXX_FLAGS_LSAN "-O0 -fsanitize=leak -DLEAK_SANITIZER") ## Use for BE-UT set(CXX_FLAGS_ASAN_UT "-O0 -fsanitize=address -DADDRESS_SANITIZER") @@ -961,6 +1086,50 @@ endif() add_subdirectory(${SRC_DIR}/util) +set(DORIS_BE_SOURCE_TARGETS + Agent + Common + Core + Exec + Exprs + Format + DorisGen + IO + Storage + Runtime + Service + doris_be + Udf + Cloud + Load + InformationSchema + meta_tool + packed_file_tool + index_tool + fs_benchmark_tool + file_cache_lru_tool + file_cache_microbench + task_executor_simulator +) +foreach(target IN LISTS DORIS_BE_SOURCE_TARGETS) + if (TARGET ${target}) + enable_doris_asan_for_sources(${target}) + endif() +endforeach() + +if (ENABLE_EXPR_ASAN OR DORIS_BE_ASAN_SOURCES) + foreach(target IN LISTS DORIS_BE_SOURCE_TARGETS) + if (TARGET ${target}) + get_target_property(target_type ${target} TYPE) + if (target_type STREQUAL "EXECUTABLE") + link_doris_asan_runtime(${target}) + endif() + endif() + endforeach() +endif() + +validate_doris_asan_sources() + # this include doris_be_test if (MAKE_TEST) include_directories(${TEST_DIR}/) diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt index d8995ccd8ad737..b0f8c10a5552c0 100644 --- a/be/src/exprs/CMakeLists.txt +++ b/be/src/exprs/CMakeLists.txt @@ -30,6 +30,9 @@ set(SRC_FILES ${SRC_FILES} ) add_library(Exprs STATIC ${SRC_FILES}) +if (ENABLE_EXPR_ASAN) + enable_doris_asan(Exprs) +endif() # function_array_distance uses faiss headers (platform_macros.h, distances.h), # which are exported by ann_index via PUBLIC linkage with faiss. target_link_libraries(Exprs PRIVATE ann_index) @@ -57,4 +60,4 @@ set(WARNING_OPTION "-Wno-unused-but-set-variable") check_cxx_compiler_flag(${WARNING_OPTION} HAS_WARNING_OPTION) if (HAS_WARNING_OPTION) target_compile_options(Exprs PRIVATE ${WARNING_OPTION} "-Wno-unused-macros") -endif() \ No newline at end of file +endif() diff --git a/build.sh b/build.sh index a5fe18b3e8dd11..c627f61c69ce81 100755 --- a/build.sh +++ b/build.sh @@ -38,6 +38,27 @@ export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib" HADOOP_DEPS_NAME="hadoop-deps" . "${DORIS_HOME}/env.sh" +ORIGINAL_BUILD_TYPE="${BUILD_TYPE:-}" +export BUILD_TYPE='Release' +echo "Force BUILD_TYPE to ${BUILD_TYPE} for test${ORIGINAL_BUILD_TYPE:+ (was ${ORIGINAL_BUILD_TYPE})}" + +ORIGINAL_EXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS:-}" +if [[ -n "${ORIGINAL_EXTRA_CXX_FLAGS}" ]]; then + FILTERED_EXTRA_CXX_FLAGS=() + read -r -a extra_cxx_flags <<< "${ORIGINAL_EXTRA_CXX_FLAGS}" + for extra_cxx_flag in "${extra_cxx_flags[@]}"; do + if [[ "${extra_cxx_flag}" == '-O1' ]]; then + continue + fi + FILTERED_EXTRA_CXX_FLAGS+=("${extra_cxx_flag}") + done + EXTRA_CXX_FLAGS="${FILTERED_EXTRA_CXX_FLAGS[*]}" + export EXTRA_CXX_FLAGS + if [[ "${EXTRA_CXX_FLAGS}" != "${ORIGINAL_EXTRA_CXX_FLAGS}" ]]; then + echo "Drop -O1 from EXTRA_CXX_FLAGS for test${ORIGINAL_EXTRA_CXX_FLAGS:+ (was ${ORIGINAL_EXTRA_CXX_FLAGS})}" + fi +fi + # ===== Build Profile ===== if [[ "${DORIS_BUILD_PROFILE}" == "1" ]]; then _BP_STATE="${DORIS_HOME}/.build_profile_state.$$" @@ -695,6 +716,8 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then echo "-- Build fs benchmark tool: ${BUILD_FS_BENCHMARK}" echo "-- Build task executor simulator: ${BUILD_TASK_EXECUTOR_SIMULATOR}" echo "-- Build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}" + echo "-- Enable Exprs ASAN: ${ENABLE_EXPR_ASAN:-OFF}" + echo "-- BE ASAN sources: ${DORIS_BE_ASAN_SOURCES:-}" mkdir -p "${CMAKE_BUILD_DIR}" cd "${CMAKE_BUILD_DIR}" @@ -719,6 +742,8 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DUSE_UNWIND="${USE_UNWIND}" \ -DDISPLAY_BUILD_TIME="${DISPLAY_BUILD_TIME}" \ -DENABLE_PCH="${ENABLE_PCH}" \ + -DENABLE_EXPR_ASAN="${ENABLE_EXPR_ASAN:-OFF}" \ + -DDORIS_BE_ASAN_SOURCES="${DORIS_BE_ASAN_SOURCES:-}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ -DUSE_AVX2="${USE_AVX2}" \ -DARM_MARCH="${ARM_MARCH}" \ diff --git a/regression-test/pipeline/common/custom_env.sh b/regression-test/pipeline/common/custom_env.sh index 8201deda51c164..161892a15857a4 100644 --- a/regression-test/pipeline/common/custom_env.sh +++ b/regression-test/pipeline/common/custom_env.sh @@ -18,3 +18,7 @@ # shellcheck disable=SC2034 BUILD_FS_BENCHMARK=ON +ENABLE_EXPR_ASAN=OFF +# The TeamCity compile stage copies this file to the repo root and only appends +# BUILD_TYPE later, so source-level ASAN selections must live here. +DORIS_BE_ASAN_SOURCES=be/src/exprs/vexpr.cpp,be/src/exec/runtime_filter/utils.cpp,be/src/exec/runtime_filter/runtime_filter_consumer.cpp diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index c9b61c6fb548b2..b8b12f477fd78f 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -253,17 +253,22 @@ trigger_build() { local COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$2}" local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$3}" local COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$4}" + local TEAMCITY_BUILD_TYPE_OVERRIDE="${TEAMCITY_BUILD_TYPE_OVERRIDE:-}" + local extra_teamcity_params="" if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMIT_ID_FROM_TRIGGER}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: add_build PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" return 1 fi local PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + if [[ -n "${TEAMCITY_BUILD_TYPE_OVERRIDE}" ]]; then + extra_teamcity_params="&name=env.BUILD_TYPE&value=${TEAMCITY_BUILD_TYPE_OVERRIDE}" + fi set -x if curl -s -X POST \ -u OneMoreChance:OneMoreChance \ -H "Content-Type:text/plain" \ -H "Accept: application/json" \ - "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.pr_num_from_trigger&value=${PULL_REQUEST_NUM:-}&name=env.commit_id_from_trigger&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times_from_trigger&value=${COMMENT_REPEAT_TIMES:-1}"; then + "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.pr_num_from_trigger&value=${PULL_REQUEST_NUM:-}&name=env.commit_id_from_trigger&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times_from_trigger&value=${COMMENT_REPEAT_TIMES:-1}${extra_teamcity_params}"; then set +x echo "INFO: Add new build to PIPELINE ${PIPELINE} of PR ${PULL_REQUEST_NUM} with COMMENT_REPEAT_TIMES ${COMMENT_REPEAT_TIMES:-1}" else