diff --git a/.clang-format-ignore b/.clang-format-ignore index 5f10e2a221203c..c59693204e6b26 100644 --- a/.clang-format-ignore +++ b/.clang-format-ignore @@ -4,6 +4,7 @@ be/src/apache-orc/* be/src/clucene/* be/src/gutil/* be/src/glibc-compatibility/* +be/src/macos_patches/* be/src/util/sse2neo.h be/src/util/sse2neon.h be/src/util/mustache/mustache.h diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 030d6463e56443..7e603545f2a945 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -48,13 +48,23 @@ jobs: - 'gensrc/proto/**' - 'gensrc/thrift/**' - - name: Ccache ${{ github.ref }} + - name: Free disk space if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} - uses: ./.github/actions/ccache-action - with: - key: BE-UT-macOS - max-size: "5G" - restore-keys: BE-UT-macOS- + run: | + # Remove unused Xcode versions to free ~10 GB + sudo rm -rf /Applications/Xcode_15*.app /Applications/Xcode_14*.app /Applications/Xcode_13*.app || true + # Remove iOS/tvOS/watchOS simulator runtimes (~5 GB) + sudo rm -rf /Library/Developer/CoreSimulator/Profiles/Runtimes || true + sudo rm -rf ~/Library/Developer/Xcode/iOS\ DeviceSupport || true + # Remove GitHub Actions hosted tool cache (Python/Node/Go/etc, ~10 GB) + sudo rm -rf /Users/runner/hostedtoolcache || true + sudo rm -rf /opt/hostedtoolcache || true + # Remove Android SDK / dotnet / Swift toolchains we don't use + sudo rm -rf /Users/runner/Library/Android || true + sudo rm -rf /usr/local/share/dotnet || true + sudo rm -rf /Library/Developer/Toolchains || true + brew cleanup --prune=all || true + df -h / - name: Run UT ${{ github.ref }} if: ${{ github.event_name == 'schedule' || steps.filter.outputs.be_changes == 'true' }} @@ -77,7 +87,7 @@ jobs: 'gettext' 'wget' 'pcre' - 'openjdk@11' + 'openjdk@17' 'maven' 'node' 'llvm@16' @@ -85,16 +95,39 @@ jobs: brew install "${cellars[@]}" || true pushd thirdparty + arch="$(uname -m)" + [[ "${arch}" == 'aarch64' ]] && arch='arm64' + file="doris-thirdparty-prebuilt-darwin-${arch}.tar.xz" branch="${{ github.base_ref }}" if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' ]]; then - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz \ - -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz + curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation/${file}" -o "${file}" else - curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz" \ - -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz + curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/${file}" -o "${file}" fi - tar -xvf doris-thirdparty-prebuilt-darwin-x86_64.tar.xz + tar -xf "${file}" + rm -f "${file}" popd - export JAVA_HOME="${JAVA_HOME_17_X64%\/}" - ./run-be-ut.sh --run -j "$(nproc)" --clean + export JAVA_HOME="$(brew --prefix openjdk@17)/libexec/openjdk.jdk/Contents/Home" + # Use RELEASE build instead of ASAN on macOS: + # 1) ASAN deadlocks at startup on macOS 15+ due to a dyld bug + # (dyld_shared_cache_iterate_text_swift), so ASAN tests cannot run. + # 2) ASAN compilation is 2-3x slower; RELEASE fits within the + # GitHub Actions 6-hour job limit. + export BUILD_TYPE_UT=RELEASE + # RELEASE defines -DNDEBUG which disables internal self-check + # methods (column_self_check, check_type_and_column, ...). Several + # tests rely on these checks returning errors. Undefine NDEBUG so + # the test behaviour matches the Linux ASAN_UT build (which also + # has NDEBUG undefined). + export EXTRA_CXX_FLAGS=-UNDEBUG + # Disable ccache: GitHub macOS runners have ~14 GB free disk and + # the local ccache (default 5 GB) competes with build artifacts. + # Since cache is not restored across runs (different build types + # and the prebuilt thirdparty already consumes ~4 GB), hit rate + # is effectively 0% and ccache only adds disk/IO overhead. + export CCACHE_DISABLE=1 + # Skip tests that require >4 GB of memory; macos-15 runners have + # only 7 GB RAM and these tests OOM-crash the test binary. + ./run-be-ut.sh --run -j "$(nproc)" --clean \ + -f '-*test_sink_large_string_data_over_4g*' diff --git a/.licenserc.yaml b/.licenserc.yaml index 90d2f778686701..d14872e663ae6d 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -80,6 +80,7 @@ header: - "be/src/util/sse2neo.h" - "be/src/util/sse2neon.h" - "be/src/util/utf8_check.cpp" + - "be/src/macos_patches/**" - "be/src/pch/*" - "be/test/data" - "be/test/expected_result" diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index a55f01fd52789b..ec2948b03bc8d5 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -260,7 +260,21 @@ set(USE_STAT64 0) set(USE_BTHREAD OFF) # Out of source build need to set the binary dir +# On macOS, clucene's src/ext/for builds bitpack_*.o via add_custom_command that +# invokes the C compiler directly with c_flags_list (derived from CMAKE_C_FLAGS). +# Such raw invocations do not pick up CMAKE_OSX_SYSROOT, so the SDK headers +# (stdio.h, ...) are not found. Inject -isysroot into CMAKE_C_FLAGS for the +# clucene subtree only, then restore. +if (APPLE) + execute_process(COMMAND xcrun --show-sdk-path + OUTPUT_VARIABLE MACOS_SDK_PATH OUTPUT_STRIP_TRAILING_WHITESPACE) + set(_saved_cmake_c_flags "${CMAKE_C_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isysroot ${MACOS_SDK_PATH}") +endif() add_subdirectory(${CONTRIB_PATH}/clucene ${PROJECT_BINARY_DIR}/clucene EXCLUDE_FROM_ALL) +if (APPLE) + set(CMAKE_C_FLAGS "${_saved_cmake_c_flags}") +endif() set(clucene_options -w -Wall) if (COMPILER_CLANG) @@ -337,6 +351,14 @@ add_compile_options(-g -fno-omit-frame-pointer $<$:-Wnon-virtual-dtor>) +if (OS_MACOSX) + # macOS CI runners (GitHub Actions macos-15) have 7 GB RAM. Linking the + # ~2 GB doris_be_test binary with full DWARF debug info triggers OOM in + # ld/dsymutil. Disable debug info on macOS; crash diagnosis can be done + # locally where memory is not constrained. + add_compile_options(-g0) +endif() + add_compile_options(-Wno-unused-parameter -Wno-sign-compare) @@ -378,6 +400,13 @@ if (COMPILER_CLANG) add_compile_options($<$:-stdlib=libc++>) add_definitions(-DUSE_LIBCPP) endif() + if (OS_MACOSX) + # Several unscoped enum values in Doris headers (bitmap_value.h BITMAP, + # memtable_memory_limiter.h NONE/SOFT/HARD) shadow values in protobuf- + # generated headers. Clang@18 started detecting these. Suppress on macOS + # for local UT builds; the test binary already suppresses via -Wno-shadow. + add_compile_options(-Wno-shadow -Wno-shadow-field) + endif() endif () add_definitions(-D__STDC_FORMAT_MACROS @@ -512,6 +541,13 @@ include_directories( ${SRC_DIR}/ ) +if (OS_MACOSX) + # Apple SDK lazy_split_view.h bug: __outer_iterator/__inner_iterator are forward-declared + # private but defined public, causing a hard error in Apple Clang 17. Patch fixes + # the forward declarations by moving them to the public section. + include_directories(BEFORE ${SRC_DIR}/macos_patches) +endif() + include_directories( SYSTEM ${COMMON_SRC_DIR} @@ -915,6 +951,13 @@ if (ENABLE_PCH) endif() if (COMPILER_CLANG) target_compile_options(pch PRIVATE -Xclang -fno-pch-timestamp) + if (OS_MACOSX) + # Suppress shadow warnings in PCH, consistent with doris_be_test's own compile options. + # bitmap_value.h has an unscoped enum value 'BITMAP' that shadows the one in + # olap_file.pb.h; Apple Clang detects this and fails the PCH compile. Scope this + # to macOS so Linux clang builds keep full shadow-warning coverage. + target_compile_options(pch PRIVATE -Wno-shadow -Wno-shadow-field) + endif() endif() endif() @@ -933,6 +976,17 @@ add_subdirectory(${SRC_DIR}/exprs) add_subdirectory(${SRC_DIR}/format) add_subdirectory(${SRC_DIR}/gen_cpp) add_subdirectory(${SRC_DIR}/io) +if(DISABLE_ANN) + # Propagate the flag as a C++ preprocessor macro so headers can guard + # FAISS-specific includes (e.g. function_array_distance.h, ann_index_writer.cpp). + # Must be set BEFORE add_subdirectory(storage/index/ann) so that the ann_index + # library itself sees the flag. + add_compile_definitions(DISABLE_ANN) +endif() +# Always build ann_index so production code (Storage, Exprs) can link against +# the ANN infrastructure classes. When DISABLE_ANN is ON the subdirectory +# skips the FAISS/OpenMP sub-libraries and guards the FAISS-specific code +# paths, producing a stub library that satisfies all symbol references. add_subdirectory(${SRC_DIR}/storage/index/ann) add_subdirectory(${SRC_DIR}/storage) add_subdirectory(${SRC_DIR}/runtime) diff --git a/be/src/common/factory_creator.h b/be/src/common/factory_creator.h index 0dea4d16b80807..d7f82c30c2020b 100644 --- a/be/src/common/factory_creator.h +++ b/be/src/common/factory_creator.h @@ -35,6 +35,21 @@ // during inherits // TODO try to allow make_unique // + +// On macOS, LLVM libc++ (both 18 and 20) has an internal compressed_pair +// static_cast issue when make_shared is used with types that inherit from +// enable_shared_from_this. Using the two-allocation form std::shared_ptr(new T) +// avoids the libc++ bug while still correctly initialising the enable_shared_from_this +// weak pointer (the shared_ptr constructor detects and handles this automatically). +// create_shared is defined inside the class body (via macro expansion), so it can +// access the private operator new of each class. +#ifdef __APPLE__ +#define _DORIS_CREATE_SHARED_IMPL(TypeName) \ + std::shared_ptr(new TypeName(std::forward(args)...)) +#else +#define _DORIS_CREATE_SHARED_IMPL(TypeName) std::make_shared(std::forward(args)...) +#endif + #define ENABLE_FACTORY_CREATOR(TypeName) \ private: \ void* operator new(std::size_t size) { \ @@ -59,7 +74,7 @@ public: } \ template \ static std::shared_ptr create_shared(Args&&... args) { \ - return std::make_shared(std::forward(args)...); \ + return _DORIS_CREATE_SHARED_IMPL(TypeName); \ } \ template \ static std::unique_ptr create_unique(Args&&... args) { \ diff --git a/be/src/core/binary_cast.hpp b/be/src/core/binary_cast.hpp index 3c8d9a50d9193b..155a03cc3df1ec 100644 --- a/be/src/core/binary_cast.hpp +++ b/be/src/core/binary_cast.hpp @@ -17,6 +17,7 @@ #pragma once +#include #include #include diff --git a/be/src/core/value/vdatetime_value.h b/be/src/core/value/vdatetime_value.h index 8d826884a24a20..533a924d04a7d1 100644 --- a/be/src/core/value/vdatetime_value.h +++ b/be/src/core/value/vdatetime_value.h @@ -841,7 +841,12 @@ class DateV2Value { requires std::is_integral_v DateV2Value(U other) = delete; +#ifndef __APPLE__ + // Apple Clang 17: a user-declared defaulted copy constructor taking a + // non-const reference (T&) breaks std::is_trivially_copyable even when + // also = default. The const T& overload below is sufficient. DateV2Value(DateV2Value& other) = default; +#endif DateV2Value(const DateV2Value& other) = default; diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt index d8995ccd8ad737..fec0567f4d6f07 100644 --- a/be/src/exprs/CMakeLists.txt +++ b/be/src/exprs/CMakeLists.txt @@ -30,8 +30,8 @@ set(SRC_FILES ${SRC_FILES} ) add_library(Exprs STATIC ${SRC_FILES}) -# function_array_distance uses faiss headers (platform_macros.h, distances.h), -# which are exported by ann_index via PUBLIC linkage with faiss. +# function_array_distance and vectorized_fn_call use ANN infrastructure types. +# ann_index is always built (with FAISS stubs when DISABLE_ANN=ON) so always link. target_link_libraries(Exprs PRIVATE ann_index) pch_reuse(Exprs) diff --git a/be/src/exprs/function/array/function_array_distance.h b/be/src/exprs/function/array/function_array_distance.h index 8749364d51ae6d..7cb4f52b77fc60 100644 --- a/be/src/exprs/function/array/function_array_distance.h +++ b/be/src/exprs/function/array/function_array_distance.h @@ -17,8 +17,43 @@ #pragma once +#ifndef DISABLE_ANN #include #include +#else +// When FAISS is disabled (e.g. macOS local UT builds), provide inline stubs +// for the FAISS functions used by L1/L2/InnerProduct distance classes and +// no-op definitions for the float-control pragmas. +#include +#include +#define FAISS_PRAGMA_IMPRECISE_FUNCTION_BEGIN +#define FAISS_PRAGMA_IMPRECISE_FUNCTION_END +namespace faiss { +inline float fvec_L1(const float* x, const float* y, std::size_t d) { + float s = 0; + for (std::size_t i = 0; i < d; ++i) { + s += std::abs(x[i] - y[i]); + } + return s; +} +inline float fvec_L2sqr(const float* x, const float* y, std::size_t d) { + float s = 0; + for (std::size_t i = 0; i < d; ++i) { + float diff = x[i] - y[i]; + s += diff * diff; + } + return s; +} +inline float fvec_inner_product(const float* x, const float* y, std::size_t d) { + float s = 0; + for (std::size_t i = 0; i < d; ++i) { + s += x[i] * y[i]; + } + return s; +} +} // namespace faiss +#endif + #include #include diff --git a/be/src/load/routine_load/kinesis_conf.cpp b/be/src/load/routine_load/kinesis_conf.cpp index 82cf35ea2744da..c009d0e656e0f5 100644 --- a/be/src/load/routine_load/kinesis_conf.cpp +++ b/be/src/load/routine_load/kinesis_conf.cpp @@ -115,7 +115,7 @@ Status KinesisConf::apply_to_get_shard_iterator_request( it = _get_shard_iterator_params.find("timestamp"); if (it != _get_shard_iterator_params.end()) { try { - request.SetTimestamp(Aws::Utils::DateTime(std::stol(it->second))); + request.SetTimestamp(Aws::Utils::DateTime(static_cast(std::stol(it->second)))); } catch (const std::exception&) { return Status::InternalError("Failed to apply get_shard_iterator.timestamp: {}", it->second); diff --git a/be/src/macos_patches/__ranges/lazy_split_view.h b/be/src/macos_patches/__ranges/lazy_split_view.h new file mode 100644 index 00000000000000..4e94b1b5b1ab26 --- /dev/null +++ b/be/src/macos_patches/__ranges/lazy_split_view.h @@ -0,0 +1,442 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H +#define _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H + +#include <__algorithm/ranges_find.h> +#include <__algorithm/ranges_mismatch.h> +#include <__assert> +#include <__concepts/constructible.h> +#include <__concepts/convertible_to.h> +#include <__concepts/derived_from.h> +#include <__config> +#include <__functional/bind_back.h> +#include <__functional/ranges_operations.h> +#include <__iterator/concepts.h> +#include <__iterator/default_sentinel.h> +#include <__iterator/incrementable_traits.h> +#include <__iterator/indirectly_comparable.h> +#include <__iterator/iter_move.h> +#include <__iterator/iter_swap.h> +#include <__iterator/iterator_traits.h> +#include <__memory/addressof.h> +#include <__ranges/access.h> +#include <__ranges/all.h> +#include <__ranges/concepts.h> +#include <__ranges/non_propagating_cache.h> +#include <__ranges/range_adaptor.h> +#include <__ranges/single_view.h> +#include <__ranges/subrange.h> +#include <__ranges/view_interface.h> +#include <__type_traits/conditional.h> +#include <__type_traits/decay.h> +#include <__type_traits/is_nothrow_constructible.h> +#include <__type_traits/maybe_const.h> +#include <__type_traits/remove_reference.h> +#include <__utility/forward.h> +#include <__utility/move.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +namespace ranges { + +template +struct __require_constant; + +template +concept __tiny_range = sized_range<_Range> && requires { + typename __require_constant::size()>; +} && (remove_reference_t<_Range>::size() <= 1); + +template + requires view<_View> && view<_Pattern> && + indirectly_comparable, iterator_t<_Pattern>, ranges::equal_to> && + (forward_range<_View> || __tiny_range<_Pattern>) +class lazy_split_view : public view_interface> { + _LIBCPP_NO_UNIQUE_ADDRESS _View __base_ = _View(); + _LIBCPP_NO_UNIQUE_ADDRESS _Pattern __pattern_ = _Pattern(); + + using _MaybeCurrent _LIBCPP_NODEBUG = + _If, __non_propagating_cache>, __empty_cache>; + _LIBCPP_NO_UNIQUE_ADDRESS _MaybeCurrent __current_ = _MaybeCurrent(); + +public: + _LIBCPP_HIDE_FROM_ABI lazy_split_view() + requires default_initializable<_View> && default_initializable<_Pattern> + = default; + + _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 lazy_split_view(_View __base, _Pattern __pattern) + : __base_(std::move(__base)), __pattern_(std::move(__pattern)) {} + + template + requires constructible_from<_View, views::all_t<_Range>> && + constructible_from<_Pattern, single_view>> + _LIBCPP_HIDE_FROM_ABI constexpr _LIBCPP_EXPLICIT_SINCE_CXX23 lazy_split_view(_Range&& __r, range_value_t<_Range> __e) + : __base_(views::all(std::forward<_Range>(__r))), __pattern_(views::single(std::move(__e))) {} + + _LIBCPP_HIDE_FROM_ABI constexpr _View base() const& + requires copy_constructible<_View> + { + return __base_; + } + _LIBCPP_HIDE_FROM_ABI constexpr _View base() && { return std::move(__base_); } + + _LIBCPP_HIDE_FROM_ABI constexpr auto begin() { + if constexpr (forward_range<_View>) { + return __outer_iterator < __simple_view<_View> && __simple_view < _Pattern >> {*this, ranges::begin(__base_)}; + } else { + __current_.__emplace(ranges::begin(__base_)); + return __outer_iterator{*this}; + } + } + + _LIBCPP_HIDE_FROM_ABI constexpr auto begin() const + requires forward_range<_View> && forward_range + { + return __outer_iterator{*this, ranges::begin(__base_)}; + } + + _LIBCPP_HIDE_FROM_ABI constexpr auto end() + requires forward_range<_View> && common_range<_View> + { + return __outer_iterator < __simple_view<_View> && __simple_view < _Pattern >> {*this, ranges::end(__base_)}; + } + + _LIBCPP_HIDE_FROM_ABI constexpr auto end() const { + if constexpr (forward_range<_View> && forward_range && common_range) { + return __outer_iterator{*this, ranges::end(__base_)}; + } else { + return default_sentinel; + } + } + +private: + template + struct __outer_iterator; + template + struct __inner_iterator; + + template + struct __outer_iterator_category {}; + + template + struct __outer_iterator_category<_Tp> { + using iterator_category = input_iterator_tag; + }; + + template + struct __outer_iterator : __outer_iterator_category<__maybe_const<_Const, _View>> { + private: + template + friend struct __inner_iterator; + friend __outer_iterator; + + using _Parent _LIBCPP_NODEBUG = __maybe_const<_Const, lazy_split_view>; + using _Base _LIBCPP_NODEBUG = __maybe_const<_Const, _View>; + + _Parent* __parent_ = nullptr; + using _MaybeCurrent _LIBCPP_NODEBUG = _If, iterator_t<_Base>, __empty_cache>; + _LIBCPP_NO_UNIQUE_ADDRESS _MaybeCurrent __current_ = _MaybeCurrent(); + bool __trailing_empty_ = false; + + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto& __current() noexcept { + if constexpr (forward_range<_View>) { + return __current_; + } else { + return *__parent_->__current_; + } + } + + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const auto& __current() const noexcept { + if constexpr (forward_range<_View>) { + return __current_; + } else { + return *__parent_->__current_; + } + } + + // Workaround for the GCC issue that doesn't allow calling `__parent_->__base_` from friend functions (because + // `__base_` is private). + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto& __parent_base() const noexcept { return __parent_->__base_; } + + public: + // using iterator_category = inherited; + using iterator_concept = conditional_t, forward_iterator_tag, input_iterator_tag>; + using difference_type = range_difference_t<_Base>; + + struct value_type : view_interface { + private: + __outer_iterator __i_ = __outer_iterator(); + + public: + _LIBCPP_HIDE_FROM_ABI value_type() = default; + _LIBCPP_HIDE_FROM_ABI constexpr explicit value_type(__outer_iterator __i) : __i_(std::move(__i)) {} + + _LIBCPP_HIDE_FROM_ABI constexpr __inner_iterator<_Const> begin() const { return __inner_iterator<_Const>{__i_}; } + _LIBCPP_HIDE_FROM_ABI constexpr default_sentinel_t end() const noexcept { return default_sentinel; } + }; + + _LIBCPP_HIDE_FROM_ABI __outer_iterator() = default; + + _LIBCPP_HIDE_FROM_ABI constexpr explicit __outer_iterator(_Parent& __parent) + requires(!forward_range<_Base>) + : __parent_(std::addressof(__parent)) {} + + _LIBCPP_HIDE_FROM_ABI constexpr __outer_iterator(_Parent& __parent, iterator_t<_Base> __current) + requires forward_range<_Base> + : __parent_(std::addressof(__parent)), __current_(std::move(__current)) {} + + _LIBCPP_HIDE_FROM_ABI constexpr __outer_iterator(__outer_iterator __i) + requires _Const && convertible_to, iterator_t<_Base>> + : __parent_(__i.__parent_), __current_(std::move(__i.__current_)) {} + + _LIBCPP_HIDE_FROM_ABI constexpr value_type operator*() const { return value_type{*this}; } + + _LIBCPP_HIDE_FROM_ABI constexpr __outer_iterator& operator++() { + const auto __end = ranges::end(__parent_->__base_); + if (__current() == __end) { + __trailing_empty_ = false; + return *this; + } + + const auto [__pbegin, __pend] = ranges::subrange{__parent_->__pattern_}; + if (__pbegin == __pend) { + // Empty pattern: split on every element in the input range + ++__current(); + + } else if constexpr (__tiny_range<_Pattern>) { + // One-element pattern: we can use `ranges::find`. + __current() = ranges::find(std::move(__current()), __end, *__pbegin); + if (__current() != __end) { + // Make sure we point to after the separator we just found. + ++__current(); + if (__current() == __end) + __trailing_empty_ = true; + } + + } else { + // General case for n-element pattern. + do { + const auto [__b, __p] = ranges::mismatch(__current(), __end, __pbegin, __pend); + if (__p == __pend) { + __current() = __b; + if (__current() == __end) { + __trailing_empty_ = true; + } + break; // The pattern matched; skip it. + } + } while (++__current() != __end); + } + + return *this; + } + + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) { + if constexpr (forward_range<_Base>) { + auto __tmp = *this; + ++*this; + return __tmp; + + } else { + ++*this; + } + } + + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __outer_iterator& __x, const __outer_iterator& __y) + requires forward_range<_Base> + { + return __x.__current_ == __y.__current_ && __x.__trailing_empty_ == __y.__trailing_empty_; + } + + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __outer_iterator& __x, default_sentinel_t) { + _LIBCPP_ASSERT_NON_NULL(__x.__parent_ != nullptr, "Cannot call comparison on a default-constructed iterator."); + return __x.__current() == ranges::end(__x.__parent_base()) && !__x.__trailing_empty_; + } + }; + + template + struct __inner_iterator_category {}; + + template + struct __inner_iterator_category<_Tp> { + using iterator_category = + _If< derived_from>::iterator_category, forward_iterator_tag>, + forward_iterator_tag, + typename iterator_traits>::iterator_category >; + }; + + template + struct __inner_iterator : __inner_iterator_category<__maybe_const<_Const, _View>> { + private: + using _Base _LIBCPP_NODEBUG = __maybe_const<_Const, _View>; + // Workaround for a GCC issue. + static constexpr bool _OuterConst = _Const; + __outer_iterator<_Const> __i_ = __outer_iterator<_OuterConst>(); + bool __incremented_ = false; + + // Note: these private functions are necessary because GCC doesn't allow calls to private members of `__i_` from + // free functions that are friends of `inner-iterator`. + + _LIBCPP_HIDE_FROM_ABI constexpr bool __is_done() const { + _LIBCPP_ASSERT_NON_NULL(__i_.__parent_ != nullptr, "Cannot call comparison on a default-constructed iterator."); + + auto [__pcur, __pend] = ranges::subrange{__i_.__parent_->__pattern_}; + auto __end = ranges::end(__i_.__parent_->__base_); + + if constexpr (__tiny_range<_Pattern>) { + const auto& __cur = __i_.__current(); + if (__cur == __end) + return true; + if (__pcur == __pend) + return __incremented_; + + return *__cur == *__pcur; + + } else { + auto __cur = __i_.__current(); + if (__cur == __end) + return true; + if (__pcur == __pend) + return __incremented_; + + do { + if (*__cur != *__pcur) + return false; + if (++__pcur == __pend) + return true; + } while (++__cur != __end); + + return false; + } + } + + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto& __outer_current() noexcept { return __i_.__current(); } + + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr const auto& __outer_current() const noexcept { + return __i_.__current(); + } + + public: + // using iterator_category = inherited; + using iterator_concept = typename __outer_iterator<_Const>::iterator_concept; + using value_type = range_value_t<_Base>; + using difference_type = range_difference_t<_Base>; + + _LIBCPP_HIDE_FROM_ABI __inner_iterator() = default; + + _LIBCPP_HIDE_FROM_ABI constexpr explicit __inner_iterator(__outer_iterator<_Const> __i) : __i_(std::move(__i)) {} + + _LIBCPP_HIDE_FROM_ABI constexpr const iterator_t<_Base>& base() const& noexcept { return __i_.__current(); } + _LIBCPP_HIDE_FROM_ABI constexpr iterator_t<_Base> base() && + requires forward_range<_View> + { + return std::move(__i_.__current()); + } + + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator*() const { return *__i_.__current(); } + + _LIBCPP_HIDE_FROM_ABI constexpr __inner_iterator& operator++() { + __incremented_ = true; + + if constexpr (!forward_range<_Base>) { + if constexpr (_Pattern::size() == 0) { + return *this; + } + } + + ++__i_.__current(); + return *this; + } + + _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) { + if constexpr (forward_range<_Base>) { + auto __tmp = *this; + ++*this; + return __tmp; + + } else { + ++*this; + } + } + + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __inner_iterator& __x, const __inner_iterator& __y) + requires forward_range<_Base> + { + return __x.__outer_current() == __y.__outer_current(); + } + + _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const __inner_iterator& __x, default_sentinel_t) { + return __x.__is_done(); + } + + _LIBCPP_HIDE_FROM_ABI friend constexpr decltype(auto) + iter_move(const __inner_iterator& __i) noexcept(noexcept(ranges::iter_move(__i.__outer_current()))) { + return ranges::iter_move(__i.__outer_current()); + } + + _LIBCPP_HIDE_FROM_ABI friend constexpr void iter_swap( + const __inner_iterator& __x, + const __inner_iterator& __y) noexcept(noexcept(ranges::iter_swap(__x.__outer_current(), __y.__outer_current()))) + requires indirectly_swappable> + { + ranges::iter_swap(__x.__outer_current(), __y.__outer_current()); + } + }; +}; + +template +lazy_split_view(_Range&&, _Pattern&&) -> lazy_split_view, views::all_t<_Pattern>>; + +template +lazy_split_view(_Range&&, + range_value_t<_Range>) -> lazy_split_view, single_view>>; + +namespace views { +namespace __lazy_split_view { +struct __fn { + template + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Range&& __range, _Pattern&& __pattern) const + noexcept(noexcept(lazy_split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern)))) + -> decltype(lazy_split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern))) { + return lazy_split_view(std::forward<_Range>(__range), std::forward<_Pattern>(__pattern)); + } + + template + requires constructible_from, _Pattern> + [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Pattern&& __pattern) const + noexcept(is_nothrow_constructible_v, _Pattern>) { + return __pipeable(std::__bind_back(*this, std::forward<_Pattern>(__pattern))); + } +}; +} // namespace __lazy_split_view + +inline namespace __cpo { +inline constexpr auto lazy_split = __lazy_split_view::__fn{}; +} // namespace __cpo +} // namespace views + +} // namespace ranges + +#endif // _LIBCPP_STD_VER >= 20 + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___RANGES_LAZY_SPLIT_VIEW_H diff --git a/be/src/storage/index/ann/CMakeLists.txt b/be/src/storage/index/ann/CMakeLists.txt index a30bfce9773d43..e4c1ef42e534de 100644 --- a/be/src/storage/index/ann/CMakeLists.txt +++ b/be/src/storage/index/ann/CMakeLists.txt @@ -15,18 +15,23 @@ # specific language governing permissions and limitations # under the License. -add_subdirectory(cmake-protect) +# Only build FAISS/OpenBLAS when DISABLE_ANN is not set +if(NOT DISABLE_ANN) + add_subdirectory(cmake-protect) +endif() # Use all .cpp files in this directory and subdirectories as sources -file(GLOB VECTOR_LIB_SRC CONFIGURE_DEPENDS +file(GLOB VECTOR_LIB_SRC CONFIGURE_DEPENDS *.cpp ann_index_result_cache/*.cpp ) -find_package(OpenMP REQUIRED) - add_library(ann_index STATIC ${VECTOR_LIB_SRC}) -target_link_libraries(ann_index PUBLIC faiss OpenMP::OpenMP_CXX) + +if(NOT DISABLE_ANN) + find_package(OpenMP REQUIRED) + target_link_libraries(ann_index PUBLIC faiss OpenMP::OpenMP_CXX) +endif() # Some header files from faiss are used by doris, they will break compile check. target_compile_options(ann_index PRIVATE -Wno-shadow-field) diff --git a/be/src/storage/index/ann/ann_index_reader.cpp b/be/src/storage/index/ann/ann_index_reader.cpp index 1bd81e3dc9c0f1..3e2cb1a50fea6b 100644 --- a/be/src/storage/index/ann/ann_index_reader.cpp +++ b/be/src/storage/index/ann/ann_index_reader.cpp @@ -33,7 +33,9 @@ #include "storage/index/ann/ann_index_result_cache/ann_index_result_cache_handle.h" #include "storage/index/ann/ann_index_writer.h" #include "storage/index/ann/ann_search_params.h" +#ifndef DISABLE_ANN #include "storage/index/ann/faiss_ann_index.h" +#endif #include "storage/index/index_file_reader.h" #include "storage/index/inverted/inverted_index_compound_reader.h" #include "util/once.h" @@ -86,6 +88,7 @@ Status AnnIndexReader::load_index(io::IOContext* io_ctx) { return Status::IOError("Failed to open index file: {}", compound_dir.error().to_string()); } +#ifndef DISABLE_ANN _vector_index = std::make_unique(); _vector_index->set_metric(_metric_type); _vector_index->set_type(_index_type); @@ -97,6 +100,9 @@ Status AnnIndexReader::load_index(io::IOContext* io_ctx) { ->set_ivfdata_cache_key_prefix( _index_file_reader->get_index_file_cache_key(&_index_meta)); RETURN_IF_ERROR(_vector_index->load(compound_dir->get())); +#else + return Status::NotSupported("ANN index is disabled in this build"); +#endif // Keep the compound directory alive. For IVF_ON_DISK the // CachedRandomAccessReader holds a cloned CSIndexInput whose // `base` raw pointer references the compound reader's underlying diff --git a/be/src/storage/index/ann/ann_index_writer.cpp b/be/src/storage/index/ann/ann_index_writer.cpp index 28d348cc319a48..43cc5d03c190a2 100644 --- a/be/src/storage/index/ann/ann_index_writer.cpp +++ b/be/src/storage/index/ann/ann_index_writer.cpp @@ -22,10 +22,13 @@ #include #include "common/cast_set.h" +#ifndef DISABLE_ANN #include "storage/index/ann/faiss_ann_index.h" +#endif #include "storage/index/inverted/inverted_index_fs_directory.h" namespace doris::segment_v2 { +#ifndef DISABLE_ANN static std::string get_or_default(const std::map& properties, const std::string& key, const std::string& default_value) { auto it = properties.find(key); @@ -34,6 +37,7 @@ static std::string get_or_default(const std::map& prop } return default_value; } +#endif AnnIndexColumnWriter::AnnIndexColumnWriter(IndexFileWriter* index_file_writer, const TabletIndex* index_meta) @@ -42,6 +46,7 @@ AnnIndexColumnWriter::AnnIndexColumnWriter(IndexFileWriter* index_file_writer, AnnIndexColumnWriter::~AnnIndexColumnWriter() {} Status AnnIndexColumnWriter::init() { +#ifndef DISABLE_ANN Result> compound_dir = _index_file_writer->open(_index_meta); if (!compound_dir.has_value()) { @@ -81,6 +86,9 @@ Status AnnIndexColumnWriter::init() { _float_array.reserve(block_size); return Status::OK(); +#else + return Status::NotSupported("ANN index is disabled in this build"); +#endif } Status AnnIndexColumnWriter::add_values(const std::string fn, const void* values, size_t count) { diff --git a/be/src/storage/index/ann/cmake-protect/CMakeLists.txt b/be/src/storage/index/ann/cmake-protect/CMakeLists.txt index 0c65e4ae170f7e..b9c70f50707b65 100644 --- a/be/src/storage/index/ann/cmake-protect/CMakeLists.txt +++ b/be/src/storage/index/ann/cmake-protect/CMakeLists.txt @@ -19,10 +19,21 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") +# Strip ASAN/UBSAN flags so that getarch (compiled and run by OpenBLAS cmake at +# configure time) does not hang during ASAN runtime initialization on macOS. +# Only on Apple: the getarch hang is specific to Apple's ASAN runtime; on Linux +# OpenBLAS builds fine with sanitizers, so keep them there for full instrumentation. +if(APPLE) + foreach(_flag CMAKE_C_FLAGS CMAKE_CXX_FLAGS CMAKE_EXE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) + string(REGEX REPLACE "-fsanitize=[^ \t]*" "" ${_flag} "${${_flag}}") + string(REPLACE "-DADDRESS_SANITIZER" "" ${_flag} "${${_flag}}") + string(REPLACE "-DUNDEFINED_SANITIZER" "" ${_flag} "${${_flag}}") + endforeach() +endif() + set(BUILD_WITHOUT_LAPACK OFF CACHE BOOL "Disable LAPACK support in OpenBLAS") set(NO_SHARED TRUE CACHE BOOL "Disable shared library in OpenBLAS") set(C_LAPACK TRUE CACHE BOOL "Enable C interface for LAPACK in OpenBLAS") -set(USE_OPENMP TRUE CACHE BOOL "Enable OpenMP in OpenBLAS") set(NOFORTRAN ON CACHE BOOL "Disable Fortran in OpenBLAS") set(BUILD_STATIC_LIBS ON CACHE BOOL "Build static libraries in OpenBLAS") set(BUILD_TESTING OFF CACHE BOOL "Build shared libraries in OpenBLAS") @@ -32,6 +43,20 @@ set(NO_LAPACK OFF CACHE BOOL "Disable LAPACK in OpenBLAS") set(NO_CBLAS ON CACHE BOOL "Disable CBLAS in OpenBLAS") set(NO_AVX512 ON CACHE BOOL "Disable AVX512 in OpenBLAS") +if(CMAKE_SYSTEM_NAME MATCHES "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64") + # On Apple Silicon: disable OpenMP in OpenBLAS (omp.h not in default include + # path) and force ARMV8 target to avoid getarch CPU-detection hang. + set(USE_OPENMP FALSE CACHE BOOL "Enable OpenMP in OpenBLAS" FORCE) + set(TARGET "ARMV8" CACHE STRING "OpenBLAS target architecture" FORCE) + # Make libomp headers visible to faiss and any other sub-target that needs them. + execute_process(COMMAND brew --prefix libomp OUTPUT_VARIABLE LIBOMP_PREFIX OUTPUT_STRIP_TRAILING_WHITESPACE) + include_directories(SYSTEM "${LIBOMP_PREFIX}/include") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -I${LIBOMP_PREFIX}/include") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${LIBOMP_PREFIX}/include") +else() + set(USE_OPENMP TRUE CACHE BOOL "Enable OpenMP in OpenBLAS") +endif() + # Allow overriding DYNAMIC_ARCH from build.sh via ENABLE_DYNAMIC_ARCH variable if (DEFINED ENABLE_DYNAMIC_ARCH) if (NOT CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64") diff --git a/be/src/storage/index/ann/faiss_ann_index.cpp b/be/src/storage/index/ann/faiss_ann_index.cpp index f933f3c683f940..e02f19eeee0f49 100644 --- a/be/src/storage/index/ann/faiss_ann_index.cpp +++ b/be/src/storage/index/ann/faiss_ann_index.cpp @@ -17,6 +17,8 @@ #include "storage/index/ann/faiss_ann_index.h" +#ifndef DISABLE_ANN + #include #include #include @@ -1146,3 +1148,5 @@ doris::Status FaissVectorIndex::load(lucene::store::Directory* dir) { } } // namespace doris::segment_v2 + +#endif // DISABLE_ANN diff --git a/be/src/storage/index/ann/faiss_ann_index.h b/be/src/storage/index/ann/faiss_ann_index.h index e1f1f65a3adadb..dadf440ff74a51 100644 --- a/be/src/storage/index/ann/faiss_ann_index.h +++ b/be/src/storage/index/ann/faiss_ann_index.h @@ -17,6 +17,8 @@ #pragma once +#ifndef DISABLE_ANN + #include #include #include @@ -303,3 +305,5 @@ class FaissVectorIndex : public VectorIndex { std::string _ivfdata_cache_key_prefix; ///< Cache key prefix for ivfdata blocks }; } // namespace doris::segment_v2 + +#endif // DISABLE_ANN diff --git a/be/src/storage/segment/segment_writer.cpp b/be/src/storage/segment/segment_writer.cpp index edf3ebc81dd0b7..bc942683cd063e 100644 --- a/be/src/storage/segment/segment_writer.cpp +++ b/be/src/storage/segment/segment_writer.cpp @@ -1252,13 +1252,5 @@ Status SegmentWriter::_generate_short_key_index(std::vectorkeys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write; -} - -inline bool SegmentWriter::_is_mow_with_cluster_key() { - return _is_mow() && !_tablet_schema->cluster_key_uids().empty(); -} - } // namespace segment_v2 } // namespace doris diff --git a/be/src/storage/segment/segment_writer.h b/be/src/storage/segment/segment_writer.h index 9b6b8b55c3aea1..ebefa5d40b5684 100644 --- a/be/src/storage/segment/segment_writer.h +++ b/be/src/storage/segment/segment_writer.h @@ -189,8 +189,12 @@ class SegmentWriter { IOlapColumnDataAccessor* seq_column, size_t num_rows, bool need_sort); Status _generate_short_key_index(std::vector& key_columns, size_t num_rows, const std::vector& short_key_pos); - bool _is_mow(); - bool _is_mow_with_cluster_key(); + bool _is_mow() { + return _tablet_schema->keys_type() == UNIQUE_KEYS && _opts.enable_unique_key_merge_on_write; + } + bool _is_mow_with_cluster_key() { + return _is_mow() && !_tablet_schema->cluster_key_uids().empty(); + } protected: // Build key index for derived writers that override append_block. diff --git a/be/src/util/md5.cpp b/be/src/util/md5.cpp index b54e4eed8de890..8603306e0045ca 100644 --- a/be/src/util/md5.cpp +++ b/be/src/util/md5.cpp @@ -31,10 +31,10 @@ namespace doris { namespace { -constexpr uint32_t MD5_A0 = 0x67452301; -constexpr uint32_t MD5_B0 = 0xefcdab89; -constexpr uint32_t MD5_C0 = 0x98badcfe; -constexpr uint32_t MD5_D0 = 0x10325476; +[[maybe_unused]] constexpr uint32_t MD5_A0 = 0x67452301; +[[maybe_unused]] constexpr uint32_t MD5_B0 = 0xefcdab89; +[[maybe_unused]] constexpr uint32_t MD5_C0 = 0x98badcfe; +[[maybe_unused]] constexpr uint32_t MD5_D0 = 0x10325476; constexpr unsigned char MD5_DUMMY_INPUT = 0; void md5_to_hex(const unsigned char* digest, char* out) { @@ -49,7 +49,8 @@ size_t md5_num_blocks(size_t len) { return (len + 9 + 63) / 64; } -size_t md5_pad_final_blocks(const unsigned char* data, size_t len, unsigned char* out) { +[[maybe_unused]] size_t md5_pad_final_blocks(const unsigned char* data, size_t len, + unsigned char* out) { size_t full_blocks = len / 64; size_t tail = len % 64; size_t num_blocks = md5_num_blocks(len); diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index d52c72b91994a6..ee047fe21e8cd3 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -23,10 +23,11 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/test") file(GLOB_RECURSE UT_FILES CONFIGURE_DEPENDS *.cpp) -# Remove all cpp files from vector search (ANN index) subdirectory -# Since cpp files in vector search subdirs use header files from faiss. -# The compile check used by doris can not be applied to faiss headers. -# So vector_search cpp files are compiled in a separate library using different compile options. +# Remove all cpp files from vector search (ANN index) subdirectory. +# When DISABLE_ANN is OFF: they are compiled in a separate library (vector_search_test) +# with relaxed compiler flags (faiss headers are sensitive to strict warnings). +# When DISABLE_ANN is ON: they are guarded by #ifndef DISABLE_ANN and produce empty +# objects, but are still excluded to keep UT_FILES clean. file(GLOB_RECURSE VECTOR_FILES CONFIGURE_DEPENDS storage/index/ann/*.cpp) list(REMOVE_ITEM UT_FILES ${VECTOR_FILES}) @@ -112,27 +113,38 @@ include_directories( ) # Removed test files are added back using separate compile arguments. -add_subdirectory(storage/index/ann) +if(NOT DISABLE_ANN) + add_subdirectory(storage/index/ann) +endif() add_executable(doris_be_test ${UT_FILES}) -if (APPLE) - target_link_libraries(doris_be_test ${TEST_LINK_LIBS} - -Wl,-force_load,$) +if(NOT DISABLE_ANN) + if (APPLE) + target_link_libraries(doris_be_test ${TEST_LINK_LIBS} + -Wl,-force_load,$) + else() + target_link_libraries(doris_be_test ${TEST_LINK_LIBS} + -Wl,--whole-archive vector_search_test -Wl,--no-whole-archive) + endif() else() - target_link_libraries(doris_be_test ${TEST_LINK_LIBS} - -Wl,--whole-archive vector_search_test -Wl,--no-whole-archive) + target_link_libraries(doris_be_test ${TEST_LINK_LIBS}) endif() set_target_properties(doris_be_test PROPERTIES COMPILE_FLAGS "-fno-access-control") target_compile_options(doris_be_test PRIVATE -include gtest/gtest.h -Wno-shadow -Wno-shadow-field) if (OS_MACOSX AND ARCH_ARM) find_program(DSYMUTIL NAMES dsymutil) - message(STATUS "dsymutil found: ${DSYMUTIL}") find_program(LLVM_STRIP NAMES llvm-strip) + message(STATUS "dsymutil found: ${DSYMUTIL}") message(STATUS "llvm-strip found: ${LLVM_STRIP}") - add_custom_command(TARGET doris_be_test POST_BUILD - COMMAND ${DSYMUTIL} $ - COMMAND ${LLVM_STRIP} --strip-all $ - ) + # Only add the POST_BUILD step when both tools are available. Without this + # guard, find_program() returns the sentinel "-NOTFOUND" which then + # gets executed verbatim by the shell and fails with exit code 127. + if (DSYMUTIL AND LLVM_STRIP) + add_custom_command(TARGET doris_be_test POST_BUILD + COMMAND ${DSYMUTIL} $ + COMMAND ${LLVM_STRIP} --strip-all $ + ) + endif() endif() diff --git a/be/test/core/column/column_array_test.cpp b/be/test/core/column/column_array_test.cpp index 7b2ec0a2544ff2..eb8bd2088fe610 100644 --- a/be/test/core/column/column_array_test.cpp +++ b/be/test/core/column/column_array_test.cpp @@ -448,7 +448,7 @@ TEST_F(ColumnArrayTest, GetDataAtTest) { TEST_F(ColumnArrayTest, FieldTest) { MutableColumns array_columns_copy; DataTypeSerDeSPtrs serdes_copy; - array_columns_copy.push_back(array_columns[42]->assume_mutable()); + array_columns_copy.push_back(array_columns[42]->assert_mutable()); serdes_copy.push_back(serdes[42]); assert_field_callback(array_columns_copy, serdes_copy); } @@ -619,7 +619,7 @@ TEST_F(ColumnArrayTest, CreateArrayTest) { // - Wrapping shared ColumnConst in ColumnArray violates use_count() assumptions in clear_column_data() for (auto& array_column : array_columns) { const auto* column = check_and_get_column( - remove_nullable(array_column->assume_mutable()).get()); + remove_nullable(array_column->assert_mutable()).get()); auto column_size = column->size(); LOG(INFO) << "column_type: " << column->get_name(); @@ -630,7 +630,7 @@ TEST_F(ColumnArrayTest, CreateArrayTest) { auto const_data = ColumnConst::create(std::move(tmp_data_col), column_size); EXPECT_ANY_THROW({ auto new_array_column = - ColumnArray::create(const_data->assume_mutable(), column->get_offsets_ptr()); + ColumnArray::create(const_data->assert_mutable(), column->get_offsets_ptr()); }); // 2. offsets_column is ColumnConst (violates check_const_only_in_top_level) @@ -639,15 +639,15 @@ TEST_F(ColumnArrayTest, CreateArrayTest) { auto const_offsets = ColumnConst::create(std::move(tmp_offsets_col), column_size); EXPECT_ANY_THROW({ auto new_array_column = - ColumnArray::create(column->get_data_ptr(), const_offsets->assume_mutable()); + ColumnArray::create(column->get_data_ptr(), const_offsets->assert_mutable()); }); // 3. offsets size does not match data size auto tmp_data_col1 = column->get_data_ptr()->clone_resized(2); EXPECT_ANY_THROW({ auto new_array_column = ColumnArray::create( - tmp_data_col1->assume_mutable(), - column->get_offsets_column().clone_resized(1)->assume_mutable()); + tmp_data_col1->assert_mutable(), + column->get_offsets_column().clone_resized(1)->assert_mutable()); }); // Test successful creation with normal columns @@ -689,7 +689,7 @@ TEST_F(ColumnArrayTest, ConvertIfOverflowAndInsertTest) { auto ptr = column->convert_column_if_overflow(); EXPECT_EQ(ptr.get(), column.get()); auto arr_col = - check_and_get_column(remove_nullable(column->assume_mutable()).get()); + check_and_get_column(remove_nullable(column->assert_mutable()).get()); auto nested_col = arr_col->get_data_ptr(); auto array_col1 = check_and_get_column(remove_nullable(ptr).get()); auto nested_col1 = array_col1->get_data_ptr(); @@ -717,7 +717,7 @@ TEST_F(ColumnArrayTest, GetNumberOfDimensionsTest) { // test dimension of array for (int i = 0; i < array_columns.size(); i++) { auto column = check_and_get_column( - remove_nullable(array_columns[i]->assume_mutable()).get()); + remove_nullable(array_columns[i]->assert_mutable()).get()); auto check_type = remove_nullable(array_types[i]); auto dimension = 0; while (check_type->get_primitive_type() == TYPE_ARRAY && !check_type->is_nullable()) { @@ -735,7 +735,7 @@ TEST_F(ColumnArrayTest, IsExclusiveTest) { auto callback = [&](const MutableColumns& columns, const DataTypeSerDeSPtrs& serdes) { for (int i = 0; i < columns.size(); i++) { auto column = check_and_get_column( - remove_nullable(columns[i]->assume_mutable()).get()); + remove_nullable(columns[i]->assert_mutable()).get()); auto cloned = columns[i]->clone_resized(1); // test expect true EXPECT_TRUE(column->is_exclusive()); @@ -759,7 +759,7 @@ TEST_F(ColumnArrayTest, MaxArraySizeAsFieldTest) { // in operator[] and get() for (int i = 0; i < array_columns.size(); i++) { auto column = check_and_get_column( - remove_nullable(array_columns[i]->assume_mutable()).get()); + remove_nullable(array_columns[i]->assert_mutable()).get()); auto check_type = remove_nullable(array_types[i]); Field a; column->get(column->size() - 1, a); @@ -794,7 +794,7 @@ TEST_F(ColumnArrayTest, IsDefaultAtTest) { // test is_default_at for (int i = 0; i < array_columns.size(); i++) { auto column = check_and_get_column( - remove_nullable(array_columns[i]->assume_mutable()).get()); + remove_nullable(array_columns[i]->assert_mutable()).get()); auto column_size = column->size(); for (int j = 0; j < column_size; j++) { auto is_default = column->is_default_at(j); @@ -818,10 +818,10 @@ TEST_F(ColumnArrayTest, HasEqualOffsetsTest) { // test has_equal_offsets which more likely used in function, eg: function_array_zip for (int i = 0; i < array_columns.size(); i++) { auto column = check_and_get_column( - remove_nullable(array_columns[i]->assume_mutable()).get()); + remove_nullable(array_columns[i]->assert_mutable()).get()); auto cloned = array_columns[i]->clone_resized(array_columns[i]->size()); auto cloned_arr = - check_and_get_column(remove_nullable(cloned->assume_mutable()).get()); + check_and_get_column(remove_nullable(cloned->assert_mutable()).get()); // test expect true EXPECT_EQ(column->get_offsets().size(), cloned_arr->get_offsets().size()); EXPECT_TRUE(column->has_equal_offsets(*cloned_arr)); diff --git a/be/test/core/column/column_check_const_only_in_top_level_test.cpp b/be/test/core/column/column_check_const_only_in_top_level_test.cpp index 8e2bb7d8c70c90..957a7d6284d7a1 100644 --- a/be/test/core/column/column_check_const_only_in_top_level_test.cpp +++ b/be/test/core/column/column_check_const_only_in_top_level_test.cpp @@ -118,8 +118,8 @@ TEST_F(ColumnCheckConstOnlyInTopLevelTest, ColumnStructWithNonConstElements) { auto int_col2 = ColumnHelper::create_column({4, 5, 6}); MutableColumns columns; - columns.push_back(int_col1->assume_mutable()); - columns.push_back(int_col2->assume_mutable()); + columns.push_back(int_col1->assert_mutable()); + columns.push_back(int_col2->assert_mutable()); // Should not throw EXPECT_NO_THROW({ diff --git a/be/test/core/column/column_ip_test.cpp b/be/test/core/column/column_ip_test.cpp index fc03446e45503b..ac604e093b9057 100644 --- a/be/test/core/column/column_ip_test.cpp +++ b/be/test/core/column/column_ip_test.cpp @@ -200,8 +200,8 @@ TEST_F(ColumnIPTest, CloneTest) { ip_cols.push_back(column_ipv4->get_ptr()); ip_cols.push_back(column_ipv6->get_ptr()); load_data_from_csv(serde, ip_cols, data_files[0], ';', {1, 2}); - assert_clone_empty(column_ipv4->assume_mutable_ref()); - assert_clone_empty(column_ipv6->assume_mutable_ref()); + assert_clone_empty(column_ipv4->assert_mutable_ref()); + assert_clone_empty(column_ipv6->assert_mutable_ref()); check_data(ip_cols, serde, ';', {1, 2}, data_files[0], assert_clone_resized_callback); } @@ -257,8 +257,8 @@ TEST_F(ColumnIPTest, PermutationAndSortTest) { ip_cols.push_back(column_ipv4->get_ptr()); ip_cols.push_back(column_ipv6->get_ptr()); load_data_from_csv(serde, ip_cols, data_files[1], ';', {1, 2}); - assert_column_permutations(column_ipv4->assume_mutable_ref(), dt_ipv4); - assert_column_permutations(column_ipv6->assume_mutable_ref(), dt_ipv6); + assert_column_permutations(column_ipv4->assert_mutable_ref(), dt_ipv4); + assert_column_permutations(column_ipv6->assert_mutable_ref(), dt_ipv6); } TEST_F(ColumnIPTest, FilterTest) { diff --git a/be/test/core/column/column_variant_test.cpp b/be/test/core/column/column_variant_test.cpp index ebf59b77345d14..51fed58007be28 100644 --- a/be/test/core/column/column_variant_test.cpp +++ b/be/test/core/column/column_variant_test.cpp @@ -1280,7 +1280,7 @@ TEST_F(ColumnVariantTest, get_data_at) { TEST_F(ColumnVariantTest, replace_column_data) { EXPECT_ANY_THROW( - column_variant->replace_column_data(column_variant->assume_mutable_ref(), 0, 0)); + column_variant->replace_column_data(column_variant->assert_mutable_ref(), 0, 0)); } TEST_F(ColumnVariantTest, serialize_value_into_arena) { @@ -3129,21 +3129,21 @@ TEST_F(ColumnVariantTest, subcolumn_operations_coverage) { col_arr->insert(an); MutableColumnPtr nested_object = ColumnVariant::create( container_variant.max_subcolumns_count(), false, col_arr->get_data().size()); - MutableColumnPtr offset = col_arr->get_offsets_ptr()->assume_mutable(); // [3, 3, 4] + MutableColumnPtr offset = col_arr->get_offsets_ptr()->assert_mutable(); // [3, 3, 4] auto* nested_object_ptr = assert_cast(nested_object.get()); // flatten nested arrays - MutableColumnPtr flattend_column = col_arr->get_data_ptr()->assume_mutable(); + MutableColumnPtr flattend_column = col_arr->get_data_ptr()->assert_mutable(); DataTypePtr flattend_type = DataTypeFactory::instance().create_data_type( FieldType::OLAP_FIELD_TYPE_BIGINT, 0, 0); // add sub path without parent prefix PathInData sub_path("k"); nested_object_ptr->add_sub_column(sub_path, std::move(flattend_column), std::move(flattend_type)); - nested_object = make_nullable(nested_object->get_ptr())->assume_mutable(); + nested_object = make_nullable(nested_object->get_ptr())->assert_mutable(); auto array = make_nullable(ColumnArray::create(std::move(nested_object), std::move(offset))); PathInData path("v.k"); - container_variant.add_sub_column(path, array->assume_mutable(), + container_variant.add_sub_column(path, array->assert_mutable(), container_variant.NESTED_TYPE); container_variant.set_num_rows(3); for (auto subcolumn : container_variant.get_subcolumns()) { diff --git a/be/test/core/column/common_column_test.h b/be/test/core/column/common_column_test.h index ac4ed5eff76582..744f4e5ff8c370 100644 --- a/be/test/core/column/common_column_test.h +++ b/be/test/core/column/common_column_test.h @@ -634,7 +634,7 @@ class CommonColumnTest : public ::testing::Test { Block block; for (size_t i = 0; i < load_cols.size(); ++i) { ColumnWithTypeAndName columnTypeAndName; - columnTypeAndName.column = load_cols[i]->assume_mutable(); + columnTypeAndName.column = load_cols[i]->assert_mutable(); columnTypeAndName.type = types[i]; block.insert(columnTypeAndName); } @@ -691,7 +691,7 @@ class CommonColumnTest : public ::testing::Test { continue; } else if (*pos + *cl > source_column->size()) { if (is_column( - remove_nullable(source_column->assume_mutable()).get())) { + remove_nullable(source_column->assert_mutable()).get())) { // insert_range_from in array has DCHECK_LG continue; } diff --git a/be/test/core/data_type/common_data_type_serder_test.h b/be/test/core/data_type/common_data_type_serder_test.h index d968cc1213e92d..49b6e0bb5e3149 100644 --- a/be/test/core/data_type/common_data_type_serder_test.h +++ b/be/test/core/data_type/common_data_type_serder_test.h @@ -277,7 +277,7 @@ class CommonDataTypeSerdeTest : public ::testing::Test { jsonb_column->reserve(load_cols[0]->size()); MutableColumns assert_cols; for (size_t i = 0; i < load_cols.size(); ++i) { - assert_cols.push_back(load_cols[i]->assume_mutable()); + assert_cols.push_back(load_cols[i]->assert_mutable()); } DataTypeSerDe::FormatOptions options; auto tz = cctz::utc_time_zone(); diff --git a/be/test/core/data_type/data_type_array_test.cpp b/be/test/core/data_type/data_type_array_test.cpp index ebc6f3eedb8d42..fd4fed449c2fbc 100644 --- a/be/test/core/data_type/data_type_array_test.cpp +++ b/be/test/core/data_type/data_type_array_test.cpp @@ -441,7 +441,7 @@ TEST_F(DataTypeArrayTest, FromAndToStringTest) { if (i == 13 || i == 31) { continue; } - assert_to_string_from_string_assert(column->assume_mutable(), type); + assert_to_string_from_string_assert(column->assert_mutable(), type); } } diff --git a/be/test/core/jsonb/jsonb_document_cast_test.cpp b/be/test/core/jsonb/jsonb_document_cast_test.cpp index 7d3f4128d91f89..e367edacf89399 100644 --- a/be/test/core/jsonb/jsonb_document_cast_test.cpp +++ b/be/test/core/jsonb/jsonb_document_cast_test.cpp @@ -105,21 +105,21 @@ class JsonbDocumentCastTest : public testing::Test { { JsonbWriter writer; - Decimal64 dec = 1234567890123456789; + Decimal64 dec = 1234567890123456789LL; writer.writeDecimal(dec, 18, 2); to_jsonbs(writer); } { JsonbWriter writer; - Decimal128V3 dec = 1234567890123456789; + Decimal128V3 dec = 1234567890123456789LL; writer.writeDecimal(dec, 38, 3); to_jsonbs(writer); } { JsonbWriter writer; - Decimal256 dec {1234567890123456789}; + Decimal256 dec {1234567890123456789LL}; writer.writeDecimal(dec, 76, 3); to_jsonbs(writer); } diff --git a/be/test/core/value/jsonb_value_test2.cpp b/be/test/core/value/jsonb_value_test2.cpp index 5b8af2458b0dfd..10fb996362cc37 100644 --- a/be/test/core/value/jsonb_value_test2.cpp +++ b/be/test/core/value/jsonb_value_test2.cpp @@ -73,7 +73,7 @@ TEST(JsonbValueConvertorTest, JsonbValueValid) { ASSERT_EQ(input->size(), 5); // 2. put column into block - ColumnWithTypeAndName argument(input->assume_mutable(), dataTypeJsonb, "jsonb_column"); + ColumnWithTypeAndName argument(input->assert_mutable(), dataTypeJsonb, "jsonb_column"); Block block; block.insert(argument); @@ -115,7 +115,7 @@ TEST(JsonbValueConvertorTest, JsonbValueValid) { ASSERT_EQ(5, nullable_col->size()); // 2. put column into block - ColumnWithTypeAndName argument1(nullable_col->assume_mutable(), nullable_dataTypeJsonb, + ColumnWithTypeAndName argument1(nullable_col->assert_mutable(), nullable_dataTypeJsonb, "jsonb_column_null"); block.clear(); block.insert(argument1); @@ -175,7 +175,7 @@ TEST(JsonbValueConvertorTest, JsonbValueInvalid) { ASSERT_EQ(input->size(), 5); // 2. put column into block - ColumnWithTypeAndName argument(input->assume_mutable(), dataTypeJsonb, "jsonb_column"); + ColumnWithTypeAndName argument(input->assert_mutable(), dataTypeJsonb, "jsonb_column"); Block block; block.insert(argument); @@ -224,7 +224,7 @@ TEST(JsonbValueConvertorTest, JsonbValueInvalid) { ASSERT_EQ(5, nullable_col->size()); // 2. put column into block - ColumnWithTypeAndName argument1(nullable_col->assume_mutable(), nullable_dataTypeJsonb, + ColumnWithTypeAndName argument1(nullable_col->assert_mutable(), nullable_dataTypeJsonb, "jsonb_column_null"); block.clear(); block.insert(argument1); diff --git a/be/test/exec/column_type_convert_test.cpp b/be/test/exec/column_type_convert_test.cpp index 5178cddbd59d2f..b4d2934f6fc33b 100644 --- a/be/test/exec/column_type_convert_test.cpp +++ b/be/test/exec/column_type_convert_test.cpp @@ -64,7 +64,7 @@ TEST_F(ColumnTypeConverterTest, TestIntegerWideningConversions) { src_data.push_back(std::numeric_limits::min()); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -97,7 +97,7 @@ TEST_F(ColumnTypeConverterTest, TestIntegerWideningConversions) { src_data.push_back(std::numeric_limits::min()); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -131,7 +131,7 @@ TEST_F(ColumnTypeConverterTest, TestIntegerNarrowingConversions) { src_data.push_back(std::numeric_limits::min()); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -161,7 +161,7 @@ TEST_F(ColumnTypeConverterTest, TestIntegerNarrowingConversions) { src_data.push_back(std::numeric_limits::min() - 1); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(!st.ok()); @@ -190,7 +190,7 @@ TEST_F(ColumnTypeConverterTest, TestFloatingPointConversions) { src_data.push_back(1L << 23); src_data.push_back((1L << 23) + 1); auto dst_nullable_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_nullable_col->assume_mutable(); + auto mutable_dst = dst_nullable_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -233,7 +233,7 @@ TEST_F(ColumnTypeConverterTest, TestFloatingPointConversions) { src_col->insert_data("", 0); // Empty string auto dst_nullable_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_nullable_col->assume_mutable(); + auto mutable_dst = dst_nullable_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -290,7 +290,7 @@ TEST_F(ColumnTypeConverterTest, TestFloatingPointConversions) { src_data.push_back(std::numeric_limits::quiet_NaN()); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -326,7 +326,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal32(-12345)); // -123.45 auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -355,7 +355,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal32(-67890)); // -678.90 auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -382,11 +382,11 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { auto& src_data = src_col->get_data(); // Add test values - src_data.push_back(Decimal64(12345678901234)); // Normal value: 1234567890.1234 - src_data.push_back(Decimal64(-98765432109876)); // Negative value: -9876543210.9876 + src_data.push_back(Decimal64(12345678901234LL)); // Normal value: 1234567890.1234 + src_data.push_back(Decimal64(-98765432109876LL)); // Negative value: -9876543210.9876 auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); // Perform conversion Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); @@ -420,7 +420,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal32(23345)); // Too large 233.45 auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -459,7 +459,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal128V3(203345)); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -497,10 +497,10 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal256(327673345)); src_data.push_back(Decimal256(655353345)); src_data.push_back(Decimal256(655363345)); - src_data.push_back(Decimal256(3333333333332345)); + src_data.push_back(Decimal256(3333333333332345LL)); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -540,7 +540,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(0); // Zero check auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -570,7 +570,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(0); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -599,7 +599,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(0); // Zero check auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -629,7 +629,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(0); // Zero check auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& null_map = nullable_col.get_null_map_data(); @@ -668,7 +668,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { src_data.push_back(Decimal64(-1000000000)); // Out of range (underflow) auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& null_map = nullable_col.get_null_map_data(); @@ -700,7 +700,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { ASSERT_EQ(3, src_data.size()); auto dst_col = nullable_dst_type->create_column(); dst_col->resize(0); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -745,7 +745,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { auto dst_col = nullable_dst_type->create_column(); dst_col->resize(0); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -793,7 +793,7 @@ TEST_F(ColumnTypeConverterTest, TestDecimalConversions) { auto dst_col = nullable_dst_type->create_column(); dst_col->resize(0); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -838,7 +838,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { src_data.push_back(0); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -870,7 +870,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { src_data.push_back(std::numeric_limits::quiet_NaN()); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -902,7 +902,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { src_col->insert_data("2147483648", 10); // Greater than INT32_MAX auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -937,7 +937,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { auto dst_col = dst_type->create_column(); dst_col->resize(0); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -974,7 +974,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { auto dst_col = dst_type->create_column(); dst_col->resize(0); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -1006,7 +1006,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { src_data.push_back(0); // false auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -1042,7 +1042,7 @@ TEST_F(ColumnTypeConverterTest, TestStringConversions) { src_col->insert_data("", 0); // Hive: null auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast( @@ -1091,7 +1091,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToIntegerTypes) { src_col->insert_data("", 0); // Empty - should be NULL auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -1138,7 +1138,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToIntegerTypes) { src_col->insert_data("123.45", 6); // Decimal - should be NULL auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -1183,7 +1183,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToIntegerTypes) { src_col->insert_data("2147483648", 10); // Overflow - should be NULL auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -1228,7 +1228,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToIntegerTypes) { src_col->insert_data("123abc", 6); // Invalid - should be NULL auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -1271,7 +1271,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToIntegerTypes) { src_col->insert_data("123e45", 6); // Scientific notation - should be NULL auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); @@ -1461,7 +1461,7 @@ TEST_F(ColumnTypeConverterTest, TestDateTimeV2ToNumericConversions) { // 2024-01-01 00:00:00.123456 auto src_col = make_datetimev2_col({{2024, 1, 1, 0, 0, 0, 123456}}); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_TRUE(st.ok()); @@ -1485,7 +1485,7 @@ TEST_F(ColumnTypeConverterTest, TestDateTimeV2ToNumericConversions) { // 3000-01-01 00:00:00.000000 auto src_col = make_datetimev2_col({{1970, 1, 1, 0, 0, 0, 0}, {3000, 1, 1, 0, 0, 0, 0}}); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& null_map = nullable_col.get_null_map_data(); null_map.resize_fill(src_col->size(), 0); @@ -1513,7 +1513,7 @@ TEST_F(ColumnTypeConverterTest, TestDateTimeV2ToNumericConversions) { // 3000-01-01 00:00:00.000000（会溢出int32） auto src_col = make_datetimev2_col({{3000, 1, 1, 0, 0, 0, 0}}); auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); ASSERT_FALSE(st.ok()); @@ -1546,7 +1546,7 @@ TEST_F(ColumnTypeConverterTest, TestDateTimeV2ToNumericConversions) { src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 14:00:00")); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& null_map = nullable_col.get_null_map_data(); null_map.resize_fill(src_col->size(), 0); @@ -1606,7 +1606,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToDateLikeConversions) { src_col->insert_data("bad-date", 8); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -1634,7 +1634,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToDateLikeConversions) { src_col->insert_data("bad-datev2", 10); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -1660,7 +1660,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToDateLikeConversions) { src_col->insert_data("bad-datetime", 12); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -1688,7 +1688,7 @@ TEST_F(ColumnTypeConverterTest, TestStringToDateLikeConversions) { src_col->insert_data("bad-datetimev2", 14); auto dst_col = nullable_dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); auto& nullable_col = static_cast(*mutable_dst); auto& nested_col = static_cast(nullable_col.get_nested_column()); auto& null_map = nullable_col.get_null_map_data(); @@ -1716,7 +1716,7 @@ TEST_F(ColumnTypeConverterTest, TestEmptyColumnConversions) { auto src_col = ColumnInt32::create(); // Empty column (no data) auto dst_col = dst_type->create_column(); - auto mutable_dst = dst_col->assume_mutable(); + auto mutable_dst = dst_col->assert_mutable(); src_col->resize(0); dst_col->resize(0); // Perform conversion diff --git a/be/test/exec/common/schema_util_test.cpp b/be/test/exec/common/schema_util_test.cpp index 273a7238fe8177..6b2b15a513d0d9 100644 --- a/be/test/exec/common/schema_util_test.cpp +++ b/be/test/exec/common/schema_util_test.cpp @@ -817,7 +817,7 @@ TEST_F(SchemaUtilTest, TestCastColumnEdgeCases) { // Test casting from variant to variant auto variant_column = ColumnVariant::create(10, false); - variant_column->create_root(nullable_array_type, nullable_array_column->assume_mutable()); + variant_column->create_root(nullable_array_type, nullable_array_column->assert_mutable()); ColumnWithTypeAndName variant_col; variant_col.type = variant_type; @@ -1330,7 +1330,7 @@ TEST_F(SchemaUtilTest, TestParseVariantColumnsWithNulls) { auto nullable_string = make_nullable(string_column->get_ptr()); auto variant_column = ColumnVariant::create(10, false); - variant_column->create_root(string_type, nullable_string->assume_mutable()); + variant_column->create_root(string_type, nullable_string->assert_mutable()); auto nullable_variant = make_nullable(variant_column->get_ptr()); block.insert({nullable_variant, variant_type, "nullable_variant"}); @@ -1947,14 +1947,14 @@ TEST_F(SchemaUtilTest, parse_and_materialize_variant_columns_ambiguous_paths) { // Prepare the variant column with the string column as root ColumnVariant::Subcolumns dynamic_subcolumns; dynamic_subcolumns.create_root( - ColumnVariant::Subcolumn(string_col->assume_mutable(), string_type, true)); + ColumnVariant::Subcolumn(string_col->assert_mutable(), string_type, true)); auto variant_col = ColumnVariant::create(0, false, std::move(dynamic_subcolumns)); auto variant_type = std::make_shared(); // Construct the block Block block; - block.insert(ColumnWithTypeAndName(variant_col->assume_mutable(), variant_type, "v")); + block.insert(ColumnWithTypeAndName(variant_col->assert_mutable(), variant_type, "v")); // The variant column is at index 0 std::vector variant_pos = {0}; diff --git a/be/test/exec/scan/scanner_context_test.cpp b/be/test/exec/scan/scanner_context_test.cpp index 14a48e37a9a7a0..51f21956a3a9a8 100644 --- a/be/test/exec/scan/scanner_context_test.cpp +++ b/be/test/exec/scan/scanner_context_test.cpp @@ -703,9 +703,9 @@ TEST_F(ScannerContextTest, get_free_block) { scanners, limit, scan_dependency, &shared_limit, nullptr, nullptr, 0, false, parallel_tasks); scanner_context->_newly_create_free_blocks_num = newly_create_free_blocks_num.get(); - scanner_context->_newly_create_free_blocks_num->set(0L); + scanner_context->_newly_create_free_blocks_num->set(int64_t {0}); scanner_context->_scanner_memory_used_counter = scanner_memory_used_counter.get(); - scanner_context->_scanner_memory_used_counter->set(0L); + scanner_context->_scanner_memory_used_counter->set(int64_t {0}); BlockUPtr block = scanner_context->get_free_block(/*force=*/true); ASSERT_NE(block, nullptr); ASSERT_TRUE(scanner_context->_newly_create_free_blocks_num->value() == 1); diff --git a/be/test/exprs/aggregate/agg_collect_test.cpp b/be/test/exprs/aggregate/agg_collect_test.cpp index d5394ab86c428e..94e0d40eb77c1a 100644 --- a/be/test/exprs/aggregate/agg_collect_test.cpp +++ b/be/test/exprs/aggregate/agg_collect_test.cpp @@ -148,14 +148,14 @@ class VAggCollectTest : public testing::Test { agg_function->merge(place, place2, _agg_arena_pool); auto column_result = ColumnArray::create(std::move(make_nullable(data_types[0]->create_column()))); - agg_function->insert_result_into(place, column_result->assume_mutable_ref()); + agg_function->insert_result_into(place, column_result->assert_mutable_ref()); EXPECT_EQ(column_result->size(), 1); EXPECT_EQ(column_result->get_offsets()[0], is_distinct(fn_name) ? input_nums : 2 * input_nums * _repeated_times); auto column_result2 = ColumnArray::create(std::move(make_nullable(data_types[0]->create_column()))); - agg_function->insert_result_into(place2, column_result2->assume_mutable_ref()); + agg_function->insert_result_into(place2, column_result2->assert_mutable_ref()); EXPECT_EQ(column_result2->size(), 1); EXPECT_EQ(column_result2->get_offsets()[0], is_distinct(fn_name) ? input_nums : input_nums * _repeated_times); diff --git a/be/test/exprs/function/cast/cast_to_string_api_test.cpp b/be/test/exprs/function/cast/cast_to_string_api_test.cpp index b8bd0c1255bee8..ab0d2b491a7aa9 100644 --- a/be/test/exprs/function/cast/cast_to_string_api_test.cpp +++ b/be/test/exprs/function/cast/cast_to_string_api_test.cpp @@ -69,22 +69,22 @@ TEST(CastToStringTest, test) { EXPECT_EQ(str, "1234567.89"); } { - Decimal64 num = -123456789012345678; + Decimal64 num = -123456789012345678LL; std::string str = CastToString::from_decimal(num, 4); EXPECT_EQ(str, "-12345678901234.5678"); } { - Decimal128V2 num = 1234567890123; + Decimal128V2 num = 1234567890123LL; std::string str = CastToString::from_decimal(num, 6); EXPECT_EQ(str, "1234.567890"); } { - Decimal128V3 num = 1234567890567890; + Decimal128V3 num = 1234567890567890LL; std::string str = CastToString::from_decimal(num, 8); EXPECT_EQ(str, "12345678.90567890"); } { - Decimal256 num {1234567890567890}; + Decimal256 num {1234567890567890LL}; std::string str = CastToString::from_decimal(num, 10); EXPECT_EQ(str, "123456.7890567890"); } diff --git a/be/test/exprs/function/function_variant_element_test.cpp b/be/test/exprs/function/function_variant_element_test.cpp index c16e2844ad9f44..239aa18d643763 100644 --- a/be/test/exprs/function/function_variant_element_test.cpp +++ b/be/test/exprs/function/function_variant_element_test.cpp @@ -40,12 +40,12 @@ TEST(function_variant_element_test, extract_from_sparse_column) { sparse_column_offsets.push_back(sparse_column_keys->size()); variant_ptr->get_subcolumn({})->insert_default(); variant_ptr->set_num_rows(1); - variant_ptr->get_doc_value_column()->assume_mutable()->resize(1); + variant_ptr->get_doc_value_column()->assert_mutable()->resize(1); ColumnPtr result; ColumnPtr index_column_ptr = ColumnString::create(); auto* index_column_ptr_mutable = - assert_cast(index_column_ptr->assume_mutable().get()); + assert_cast(index_column_ptr->assert_mutable().get()); index_column_ptr_mutable->insert_data("profile", 7); ColumnPtr index_column = ColumnConst::create(index_column_ptr, 1); auto status = diff --git a/be/test/exprs/vexpr_test.cpp b/be/test/exprs/vexpr_test.cpp index c2269efb39b608..a8abce71315338 100644 --- a/be/test/exprs/vexpr_test.cpp +++ b/be/test/exprs/vexpr_test.cpp @@ -777,8 +777,8 @@ TEST(TEST_VEXPR, LITERALTEST) { auto src_col = ColumnDecimal128V3::create(38, 6); auto& src_data = src_col->get_data(); src_data.resize(0); - src_data.push_back(Decimal128V3(123456789012345)); - src_data.push_back(Decimal128V3(-123456789012345)); + src_data.push_back(Decimal128V3(123456789012345LL)); + src_data.push_back(Decimal128V3(-123456789012345LL)); { auto node = std::make_shared( create_texpr_node_from(src_col->operator[](0), TYPE_DECIMAL128I, 38, 6), true); diff --git a/be/test/format/native/native_reader_writer_test.cpp b/be/test/format/native/native_reader_writer_test.cpp index 5d1d7dc207cef7..c20536af2c5aaa 100644 --- a/be/test/format/native/native_reader_writer_test.cpp +++ b/be/test/format/native/native_reader_writer_test.cpp @@ -98,7 +98,7 @@ static void fill_array_column(Block& block, size_t rows) { auto& array_col = assert_cast(nested); auto& offsets = array_col.get_offsets(); auto& data = array_col.get_data(); - auto mutable_data = data.assume_mutable(); + auto mutable_data = data.assert_mutable(); for (size_t i = 0; i < rows; ++i) { if (i % 5 == 0) { @@ -144,8 +144,8 @@ static void fill_map_column(Block& block, size_t rows) { auto& keys = assert_cast(nested).get_keys(); auto& values = assert_cast(nested).get_values(); - auto mutable_keys = keys.assume_mutable(); - auto mutable_values = values.assume_mutable(); + auto mutable_keys = keys.assert_mutable(); + auto mutable_values = values.assert_mutable(); std::string k1 = "k" + std::to_string(i); std::string k2 = "k" + std::to_string(i + 1); @@ -187,8 +187,8 @@ static void fill_struct_column(Block& block, size_t rows) { nullable_col->insert_default(); } else { null_map.push_back(0); - auto mutable_field0 = fields[0]->assume_mutable(); - auto mutable_field1 = fields[1]->assume_mutable(); + auto mutable_field0 = fields[0]->assert_mutable(); + auto mutable_field1 = fields[1]->assert_mutable(); // int field mutable_field0->insert(Field::create_field( static_cast(i * 100))); @@ -495,7 +495,7 @@ TEST_F(NativeReaderWriterTest, round_trip_native_file_large_rows) { for (size_t col = 0; col < src_block.columns(); ++col) { const auto& src_col = *src_block.get_by_position(col).column; const auto& dst_col_holder = *sub_block.get_by_position(col).column; - auto dst_mutable = dst_col_holder.assume_mutable(); + auto dst_mutable = dst_col_holder.assert_mutable(); dst_mutable->insert_range_from(src_col, offset, len); } st = transformer.write(sub_block); @@ -1068,7 +1068,7 @@ static Block create_all_types_test_block() { auto& array_col = assert_cast(nested); auto& offsets = array_col.get_offsets(); auto& data = array_col.get_data(); - auto mutable_data = data.assume_mutable(); + auto mutable_data = data.assert_mutable(); mutable_data->insert( Field::create_field(static_cast(10))); @@ -1097,8 +1097,8 @@ static Block create_all_types_test_block() { auto& keys = assert_cast(nested).get_keys(); auto& values = assert_cast(nested).get_values(); - auto mutable_keys = keys.assume_mutable(); - auto mutable_values = values.assume_mutable(); + auto mutable_keys = keys.assert_mutable(); + auto mutable_values = values.assert_mutable(); mutable_keys->insert(Field::create_field(std::string("key1"))); mutable_values->insert( @@ -1129,8 +1129,8 @@ static Block create_all_types_test_block() { auto& struct_col = assert_cast(nested); const auto& fields = struct_col.get_columns(); - auto mutable_field0 = fields[0]->assume_mutable(); - auto mutable_field1 = fields[1]->assume_mutable(); + auto mutable_field0 = fields[0]->assert_mutable(); + auto mutable_field1 = fields[1]->assert_mutable(); mutable_field0->insert( Field::create_field(static_cast(999))); diff --git a/be/test/format/orc/orc_read_lines.cpp b/be/test/format/orc/orc_read_lines.cpp index c920e091821435..550e9ac28a61a4 100644 --- a/be/test/format/orc/orc_read_lines.cpp +++ b/be/test/format/orc/orc_read_lines.cpp @@ -157,7 +157,7 @@ static void read_orc_line(int64_t line, std::string block_dump, } auto data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_VARCHAR, false); - block->insert(ColumnWithTypeAndName(data_type->create_column()->assume_mutable(), data_type, + block->insert(ColumnWithTypeAndName(data_type->create_column()->assert_mutable(), data_type, "row_id")); bool eof = false; diff --git a/be/test/format/orc/orc_reader_fill_data_test.cpp b/be/test/format/orc/orc_reader_fill_data_test.cpp index eab2b97e38a3bd..0979cd13b5270f 100644 --- a/be/test/format/orc/orc_reader_fill_data_test.cpp +++ b/be/test/format/orc/orc_reader_fill_data_test.cpp @@ -82,7 +82,7 @@ TEST_F(OrcReaderFillDataTest, TestFillLongColumn) { TFileRangeDesc range; auto reader = OrcReader::create_unique(params, range, 4064, "", nullptr, nullptr, true); - MutableColumnPtr xx = column->assume_mutable(); + MutableColumnPtr xx = column->assert_mutable(); Status status = reader->_fill_doris_data_column( "test_long", xx, data_type, const_node, orc_type_ptr.get(), batch.get(), values.size()); @@ -108,7 +108,7 @@ TEST_F(OrcReaderFillDataTest, TestFillLongColumnWithNull) { TFileRangeDesc range; auto reader = OrcReader::create_unique(params, range, 4064, "", nullptr, nullptr, true); - MutableColumnPtr xx = column->assume_mutable(); + MutableColumnPtr xx = column->assert_mutable(); Status status = reader->_fill_doris_data_column("test_long_with_null", xx, data_type, const_node, @@ -166,7 +166,7 @@ TEST_F(OrcReaderFillDataTest, ComplexTypeConversionTest) { std::vector { std::make_shared(std::make_shared())}, std::vector {"col1"}); - MutableColumnPtr doris_column = doris_struct_type->create_column()->assume_mutable(); + MutableColumnPtr doris_column = doris_struct_type->create_column()->assert_mutable(); Status status = reader->_fill_doris_data_column("test", doris_column, doris_struct_type, const_node, @@ -252,7 +252,7 @@ TEST_F(OrcReaderFillDataTest, ComplexTypeConversionTest) { std::vector {std::make_shared(), std::make_shared()}, std::vector {"col1", "col2"}); - MutableColumnPtr doris_column = doris_struct_type->create_column()->assume_mutable(); + MutableColumnPtr doris_column = doris_struct_type->create_column()->assert_mutable(); Status status = reader->_fill_doris_data_column("test", doris_column, doris_struct_type, const_node, @@ -337,7 +337,7 @@ TEST_F(OrcReaderFillDataTest, ComplexTypeConversionTest) { auto doris_struct_type = std::make_shared( std::vector {std::make_shared(18, 5)}, std::vector {"col1"}); - MutableColumnPtr doris_column = doris_struct_type->create_column()->assume_mutable(); + MutableColumnPtr doris_column = doris_struct_type->create_column()->assert_mutable(); reader->_decimal_scale_params.resize(0); reader->_decimal_scale_params_index = 0; Status status = reader->_fill_doris_data_column("test", doris_column, @@ -450,7 +450,7 @@ TEST_F(OrcReaderFillDataTest, ComplexTypeConversionTest) { auto doris_struct_type = std::make_shared(std::make_shared(), std::make_shared()); - MutableColumnPtr doris_column = doris_struct_type->create_column()->assume_mutable(); + MutableColumnPtr doris_column = doris_struct_type->create_column()->assert_mutable(); Status status = reader->_fill_doris_data_column("test", doris_column, doris_struct_type, diff --git a/be/test/format/parquet/parquet_read_lines.cpp b/be/test/format/parquet/parquet_read_lines.cpp index 7ca86957d49639..9dc6ba41a2fa40 100644 --- a/be/test/format/parquet/parquet_read_lines.cpp +++ b/be/test/format/parquet/parquet_read_lines.cpp @@ -170,7 +170,7 @@ static void read_parquet_lines(std::vector numeric_types, auto data_type = DataTypeFactory::instance().create_data_type(PrimitiveType::TYPE_VARCHAR, false); - block->insert(ColumnWithTypeAndName(data_type->create_column()->assume_mutable(), data_type, + block->insert(ColumnWithTypeAndName(data_type->create_column()->assert_mutable(), data_type, "row_id")); bool eof = false; diff --git a/be/test/format/parquet/parquet_thrift_test.cpp b/be/test/format/parquet/parquet_thrift_test.cpp index 7171fe3b63cd16..fa9a848f79fafb 100644 --- a/be/test/format/parquet/parquet_thrift_test.cpp +++ b/be/test/format/parquet/parquet_thrift_test.cpp @@ -221,7 +221,7 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column const_cast(static_cast(src_column.get())); data_column = nullable_column->get_nested_column_ptr(); } else { - data_column = src_column->assume_mutable(); + data_column = src_column->assert_mutable(); } FilterMap filter_map; RETURN_IF_ERROR(filter_map.init(nullptr, 0, false)); diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 544455937a1a0c..cf99dc6fdcd036 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -4374,7 +4374,7 @@ TEST_F(BlockFileCacheTest, test_align_size) { std::random_device rd; // a seed source for the random number engine std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() std::uniform_int_distribution<> distrib(0, 10_mb + 10086); - std::ranges::for_each(std::ranges::iota_view {0, 1000}, [&](int) { + for (int i = 0; i < 1000; ++i) { size_t read_size = distrib(gen) % 1_mb; size_t read_offset = distrib(gen); auto [offset, size] = @@ -4382,7 +4382,7 @@ TEST_F(BlockFileCacheTest, test_align_size) { EXPECT_EQ(offset % 1_mb, 0); EXPECT_GE(size, 1_mb); EXPECT_LE(size, 2_mb); - }); + } } TEST_F(BlockFileCacheTest, remove_if_cached_when_isnt_releasable) { @@ -4484,7 +4484,7 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_opt_lock) { std::random_device rd; // a seed source for the random number engine std::mt19937 gen(rd()); // mersenne_twister_engine seeded with rd() std::uniform_int_distribution<> distrib(1_mb, 7_mb); - std::ranges::for_each(std::ranges::iota_view {0, 1000}, [&](int) { + for (int i = 0; i < 1000; ++i) { size_t read_offset = distrib(gen); size_t read_size = distrib(gen) % 1_mb; if (read_offset + read_size > 7_mb || read_size == 0) { @@ -4508,7 +4508,7 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_opt_lock) { } else { EXPECT_EQ(std::string(read_size, '0' + num), buffer); } - }); + } } { FileReaderSPtr local_reader; diff --git a/be/test/io/cache/block_file_cache_test_common.h b/be/test/io/cache/block_file_cache_test_common.h index de29f5543d791c..549f1a46ab886a 100644 --- a/be/test/io/cache/block_file_cache_test_common.h +++ b/be/test/io/cache/block_file_cache_test_common.h @@ -45,7 +45,6 @@ #include #include #include -#include #include #include #include diff --git a/be/test/runtime/runtime_profile_test.cpp b/be/test/runtime/runtime_profile_test.cpp index 7268146e4cdf81..0ee5fafb729f5f 100644 --- a/be/test/runtime/runtime_profile_test.cpp +++ b/be/test/runtime/runtime_profile_test.cpp @@ -56,7 +56,7 @@ TEST(RuntimeProfileTest, Basic) { counter_a->update(10); counter_a->update(-5); EXPECT_EQ(counter_a->value(), 5); - counter_a->set(1L); + counter_a->set(int64_t {1}); EXPECT_EQ(counter_a->value(), 1); counter_b = profile_a2.add_counter("B", TUnit::BYTES); @@ -129,7 +129,7 @@ TEST(RuntimeProfileTest, ProtoBasic) { counter_a->update(10); counter_a->update(-5); EXPECT_EQ(counter_a->value(), 5); - counter_a->set(1L); + counter_a->set(int64_t {1}); EXPECT_EQ(counter_a->value(), 1); counter_b = profile_a2.add_counter("B", TUnit::BYTES); @@ -421,7 +421,7 @@ TEST(RuntimeProfileTest, DerivedCounters) { RuntimeProfile::Counter* bytes_counter = profile.add_counter("bytes", TUnit::BYTES); RuntimeProfile::Counter* ticks_counter = profile.add_counter("ticks", TUnit::TIME_NS); // set to 1 sec - ticks_counter->set(1000L * 1000L * 1000L); + ticks_counter->set(int64_t {1000LL * 1000LL * 1000LL}); RuntimeProfile::DerivedCounter* throughput_counter = profile.add_derived_counter( "throughput", TUnit::BYTES, @@ -430,9 +430,9 @@ TEST(RuntimeProfileTest, DerivedCounters) { }, RuntimeProfile::ROOT_COUNTER); - bytes_counter->set(10L); + bytes_counter->set(int64_t {10}); EXPECT_EQ(throughput_counter->value(), 10); - bytes_counter->set(20L); + bytes_counter->set(int64_t {20}); EXPECT_EQ(throughput_counter->value(), 20); ticks_counter->set(ticks_counter->value() / 2); EXPECT_EQ(throughput_counter->value(), 40); @@ -523,7 +523,7 @@ TEST(RuntimeProfileTest, ProtoInfoStringTest) { // TEST(RuntimeProfileTest, AddSameCounter) { // RuntimeProfile profile("Profile"); // RuntimeProfile::Counter* counter = profile.add_counter("counter", TUnit::UNIT); -// counter->set(10L); +// counter->set(int64_t {10}); // EXPECT_EQ(counter->value(), 10L); // // Adding the same counter again should fail. diff --git a/be/test/storage/compaction/compaction_permit_limiter_test.cpp b/be/test/storage/compaction/compaction_permit_limiter_test.cpp index 8dba493c9ccc92..f6595608fe6479 100644 --- a/be/test/storage/compaction/compaction_permit_limiter_test.cpp +++ b/be/test/storage/compaction/compaction_permit_limiter_test.cpp @@ -23,6 +23,8 @@ #include #include +#include + #include "common/config.h" namespace doris { diff --git a/be/test/storage/index/ann/ann_index_edge_case_test.cpp b/be/test/storage/index/ann/ann_index_edge_case_test.cpp index aa6d7637dd4235..a1382a8f376ae8 100644 --- a/be/test/storage/index/ann/ann_index_edge_case_test.cpp +++ b/be/test/storage/index/ann/ann_index_edge_case_test.cpp @@ -40,8 +40,8 @@ TEST_F(VectorSearchTest, TestAnnIndexStatsInitialization) { EXPECT_EQ(stats.load_index_costs_ns.value(), 0); // Test setting values - stats.search_costs_ns.set(1000L); - stats.load_index_costs_ns.set(2000L); + stats.search_costs_ns.set(int64_t {1000}); + stats.load_index_costs_ns.set(int64_t {2000}); EXPECT_EQ(stats.search_costs_ns.value(), 1000); EXPECT_EQ(stats.load_index_costs_ns.value(), 2000); @@ -49,8 +49,8 @@ TEST_F(VectorSearchTest, TestAnnIndexStatsInitialization) { TEST_F(VectorSearchTest, TestAnnIndexStatsCopyConstructor) { doris::segment_v2::AnnIndexStats original; - original.search_costs_ns.set(1500L); - original.load_index_costs_ns.set(2500L); + original.search_costs_ns.set(int64_t {1500}); + original.load_index_costs_ns.set(int64_t {2500}); doris::segment_v2::AnnIndexStats copied(original); diff --git a/be/test/storage/index/ann/ann_index_iterator_test.cpp b/be/test/storage/index/ann/ann_index_iterator_test.cpp index 91f6e5dbffb1ce..81005e589aeb88 100644 --- a/be/test/storage/index/ann/ann_index_iterator_test.cpp +++ b/be/test/storage/index/ann/ann_index_iterator_test.cpp @@ -28,6 +28,8 @@ #include "storage/index/ann/faiss_ann_index.h" #include "storage/index/ann/vector_search_utils.h" +#ifndef DISABLE_ANN + using namespace doris::vector_search_utils; namespace doris::segment_v2 { @@ -339,3 +341,5 @@ TEST_F(AnnIndexIteratorTest, TestSuccessfulWorkflow) { } } // namespace doris::segment_v2 + +#endif // DISABLE_ANN diff --git a/be/test/storage/index/ann/ann_index_reader_test.cpp b/be/test/storage/index/ann/ann_index_reader_test.cpp index b193f355c2ac61..773c2b950fe567 100644 --- a/be/test/storage/index/ann/ann_index_reader_test.cpp +++ b/be/test/storage/index/ann/ann_index_reader_test.cpp @@ -32,6 +32,8 @@ #include "storage/index/ann/vector_search_utils.h" #include "storage/tablet/tablet_schema.h" +#ifndef DISABLE_ANN + using namespace doris::vector_search_utils; namespace doris { @@ -649,3 +651,5 @@ TEST_F(AnnIndexReaderTest, AnnIndexReaderIVFRangeSearch) { } } // namespace doris + +#endif // DISABLE_ANN diff --git a/be/test/storage/index/ann/ann_index_smoke_test.cpp b/be/test/storage/index/ann/ann_index_smoke_test.cpp index 3b5886cdffc456..dde8114bade0b0 100644 --- a/be/test/storage/index/ann/ann_index_smoke_test.cpp +++ b/be/test/storage/index/ann/ann_index_smoke_test.cpp @@ -30,6 +30,8 @@ #include "storage/index/index_file_writer.h" #include "storage/olap_common.h" +#ifndef DISABLE_ANN + using namespace doris::vector_search_utils; namespace doris { @@ -145,4 +147,6 @@ TEST_F(AnnIndexTest, SmokeTest) { } // namespace doris -} // namespace doris \ No newline at end of file +} // namespace doris + +#endif // DISABLE_ANN \ No newline at end of file diff --git a/be/test/storage/index/ann/ann_index_writer_test.cpp b/be/test/storage/index/ann/ann_index_writer_test.cpp index bb30f9e19794af..af5452c6be58da 100644 --- a/be/test/storage/index/ann/ann_index_writer_test.cpp +++ b/be/test/storage/index/ann/ann_index_writer_test.cpp @@ -176,7 +176,6 @@ TEST_F(AnnIndexWriterTest, TestAddArrayValuesSuccess) { ASSERT_TRUE(writer->init().ok()); // Prepare test data - const size_t dim = 4; const size_t num_rows = 3; std::vector vectors = { 1.0f, 2.0f, 3.0f, 4.0f, // Row 0 @@ -313,7 +312,6 @@ TEST_F(AnnIndexWriterTest, TestFinish) { ASSERT_TRUE(writer->init().ok()); // Add some test data before finishing - const size_t dim = 4; const size_t num_rows = 2; std::vector vectors = { 1.0f, 2.0f, 3.0f, 4.0f, // Row 0 @@ -344,7 +342,6 @@ TEST_F(AnnIndexWriterTest, TestFullWorkflow) { ASSERT_TRUE(writer->init().ok()); // 2. Add multiple batches of data - const size_t dim = 4; // Batch 1 { @@ -392,7 +389,7 @@ TEST_F(AnnIndexWriterTest, TestInvalidIndexType) { EXPECT_CALL(*_index_file_writer, open(testing::_)).WillOnce(testing::Return(fs_dir)); // This should throw an exception due to invalid index type - EXPECT_THROW(writer->init(), doris::Exception); + EXPECT_THROW((void)writer->init(), doris::Exception); } TEST_F(AnnIndexWriterTest, TestInvalidMetricType) { @@ -412,7 +409,7 @@ TEST_F(AnnIndexWriterTest, TestInvalidMetricType) { EXPECT_CALL(*_index_file_writer, open(testing::_)).WillOnce(testing::Return(fs_dir)); // This should throw an exception due to invalid metric type - EXPECT_THROW(writer->init(), doris::Exception); + EXPECT_THROW((void)writer->init(), doris::Exception); } TEST_F(AnnIndexWriterTest, TestAddMoreThanChunkSize) { @@ -436,7 +433,6 @@ TEST_F(AnnIndexWriterTest, TestAddMoreThanChunkSize) { EXPECT_CALL(*mock_index, save(testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); // CHUNK_SIZE = 10 - const size_t dim = 4; { const size_t num_rows = 6; @@ -550,7 +546,6 @@ TEST_F(AnnIndexWriterTest, TestAddArrayValuesIVF) { ASSERT_TRUE(writer->init().ok()); // Prepare test data - const size_t dim = 4; const size_t num_rows = 3; std::vector vectors = { 1.0f, 2.0f, 3.0f, 4.0f, // Row 0 @@ -596,7 +591,6 @@ TEST_F(AnnIndexWriterTest, TestAddMoreThanChunkSizeIVF) { EXPECT_CALL(*mock_index, save(testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); // CHUNK_SIZE = 10 - const size_t dim = 4; { const size_t num_rows = 6; @@ -670,8 +664,6 @@ TEST_F(AnnIndexWriterTest, TestSkipTrainWhenRemainderLessThanNlist) { EXPECT_CALL(*mock_index, add(2, testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); EXPECT_CALL(*mock_index, save(testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); - const size_t dim = 4; - // Add 12 rows total { const size_t num_rows = 10; @@ -743,8 +735,6 @@ TEST_F(AnnIndexWriterTest, TestLargeDataVolumeWithRemainderSkip) { EXPECT_CALL(*mock_index, add(3, testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); EXPECT_CALL(*mock_index, save(testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); - const size_t dim = 4; - // Add 3 batches: 10 + 10 + 3 = 23 rows for (int batch = 0; batch < 2; ++batch) { const size_t num_rows = 10; @@ -817,8 +807,6 @@ TEST_F(AnnIndexWriterTest, TestLargeDataVolumeSkipRemainder) { EXPECT_CALL(*mock_index, add(2, testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); EXPECT_CALL(*mock_index, save(testing::_)).Times(1).WillOnce(testing::Return(Status::OK())); - const size_t dim = 4; - // Add 2 batches of 10 rows for (int batch = 0; batch < 2; ++batch) { const size_t num_rows = 10; @@ -886,8 +874,6 @@ TEST_F(AnnIndexWriterTest, TestSkipIndexWhenTotalRowsLessThanNlist) { EXPECT_CALL(*mock_index, add(testing::_, testing::_)).Times(0); EXPECT_CALL(*mock_index, save(testing::_)).Times(0); - const size_t dim = 4; - // Add 3 rows { const size_t num_rows = 3; diff --git a/be/test/storage/index/ann/ann_range_search_test.cpp b/be/test/storage/index/ann/ann_range_search_test.cpp index aa9868350fb196..24264fb23dbdf5 100644 --- a/be/test/storage/index/ann/ann_range_search_test.cpp +++ b/be/test/storage/index/ann/ann_range_search_test.cpp @@ -45,6 +45,8 @@ #include "storage/segment/column_reader.h" #include "storage/segment/virtual_column_iterator.h" +#ifndef DISABLE_ANN + namespace doris { // select id, value,l2_distance_approximate(embedding, [1, 2, 3, 4, 5, 6, 7, 20]) as dist from ann_with_fulltext where l2_distance_approximate(embedding, [1, 2, 3, 4, 5, 6, 7, 20]) >= 10; @@ -820,3 +822,5 @@ TEST_F(VectorSearchTest, TestPrepareAnnRangeSearch_EarlyReturn_NonLiteralRight) } } // namespace doris + +#endif // DISABLE_ANN diff --git a/be/test/storage/index/ann/faiss_vector_index_test.cpp b/be/test/storage/index/ann/faiss_vector_index_test.cpp index 60c89951cb5e5b..3d1d99e11f9787 100644 --- a/be/test/storage/index/ann/faiss_vector_index_test.cpp +++ b/be/test/storage/index/ann/faiss_vector_index_test.cpp @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#ifndef DISABLE_ANN + #include #include #include @@ -1540,3 +1542,5 @@ TEST_F(VectorSearchTest, IVFOnDiskConcurrentSearchStampedeProtection) { } } // namespace doris + +#endif // DISABLE_ANN diff --git a/be/test/storage/index/ann/vector_search_utils.cpp b/be/test/storage/index/ann/vector_search_utils.cpp index bc3ea659837308..6ef3ec975c7e20 100644 --- a/be/test/storage/index/ann/vector_search_utils.cpp +++ b/be/test/storage/index/ann/vector_search_utils.cpp @@ -17,6 +17,8 @@ #include "storage/index/ann/vector_search_utils.h" +#ifndef DISABLE_ANN + #include #include @@ -287,3 +289,5 @@ create_tmp_ann_index_reader(std::map properties) { return std::make_pair(std::move(mock_tablet_index), ann_reader); } } // namespace doris::vector_search_utils + +#endif // DISABLE_ANN diff --git a/be/test/storage/index/ann/vector_search_utils.h b/be/test/storage/index/ann/vector_search_utils.h index 770129e72617ac..1b3549193398b5 100644 --- a/be/test/storage/index/ann/vector_search_utils.h +++ b/be/test/storage/index/ann/vector_search_utils.h @@ -46,10 +46,13 @@ #include "storage/tablet/tablet_schema.h" // Add CLucene RAM Directory header #include +#ifndef DISABLE_ANN #include +#endif using doris::segment_v2::DorisCompoundReader; +#ifndef DISABLE_ANN namespace faiss { struct Index; struct IndexHNSWFlat; @@ -58,6 +61,7 @@ struct IndexHNSWFlat; namespace doris::segment_v2 { class FaissVectorIndex; } +#endif // DISABLE_ANN namespace doris::vector_search_utils { @@ -68,6 +72,7 @@ std::vector> generate_test_vectors_matrix(int num_vectors, in // Generate random vectors as a flatten vector std::vector generate_test_vectors_flatten(int num_vectors, int dimension); +#ifndef DISABLE_ANN // Enum for different index types enum class IndexType { FLAT_L2, @@ -112,6 +117,7 @@ std::vector> perform_native_index_range_search(faiss::Inde std::unique_ptr perform_doris_index_range_search( segment_v2::VectorIndex* index, const float* query_vector, float radius, const segment_v2::IndexSearchParameters& params); +#endif // DISABLE_ANN class MockIndexFileReader : public ::doris::segment_v2::IndexFileReader { public: diff --git a/be/test/storage/iterator/vertical_merge_iterator_test.cpp b/be/test/storage/iterator/vertical_merge_iterator_test.cpp index 942721c85d9ec1..2fabcb69e38f16 100644 --- a/be/test/storage/iterator/vertical_merge_iterator_test.cpp +++ b/be/test/storage/iterator/vertical_merge_iterator_test.cpp @@ -83,7 +83,7 @@ class SparseColumnOptimizationTest : public ::testing::Test { // Simulate copy_rows logic for nullable columns with sparse optimization static void copy_rows_with_optimization(const ColumnNullable* src, size_t start, size_t count, IColumn* dst_col) { - auto* dst_mut = dst_col->assume_mutable().get(); + auto* dst_mut = dst_col->assert_mutable().get(); const size_t non_null_count = count_non_null(src, start, count); @@ -112,7 +112,7 @@ class SparseColumnOptimizationTest : public ::testing::Test { // Original copy_rows logic (direct copy) static void copy_rows_original(const IColumn* src, size_t start, size_t count, IColumn* dst_col) { - dst_col->assume_mutable()->insert_range_from(*src, start, count); + dst_col->assert_mutable()->insert_range_from(*src, start, count); } // Helper to compare two nullable columns diff --git a/be/test/storage/segment/inverted_index_reader_test.cpp b/be/test/storage/segment/inverted_index_reader_test.cpp index 982d2c3f14155f..763b8ddc3c98f4 100644 --- a/be/test/storage/segment/inverted_index_reader_test.cpp +++ b/be/test/storage/segment/inverted_index_reader_test.cpp @@ -3500,8 +3500,10 @@ class InvertedIndexReaderTest : public testing::Test { EXPECT_TRUE(status.ok()) << status; for (const auto& value : values) { - status = column_writer->add_values(column.name(), reinterpret_cast(&value), - 1); + // vector elements are proxies; copy to a local to get a real address + const T local_value = value; + status = column_writer->add_values(column.name(), + reinterpret_cast(&local_value), 1); EXPECT_TRUE(status.ok()) << status; } diff --git a/be/test/storage/segment/variant_column_writer_reader_test.cpp b/be/test/storage/segment/variant_column_writer_reader_test.cpp index 3a644cc373f896..6289f911ca71c5 100644 --- a/be/test/storage/segment/variant_column_writer_reader_test.cpp +++ b/be/test/storage/segment/variant_column_writer_reader_test.cpp @@ -3712,7 +3712,7 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) { // fill with nullable ColumnVariant target MutableColumnPtr new_column_object1 = ColumnVariant::create(3, false); MutableColumnPtr null_object = - ColumnNullable::create(new_column_object1->assume_mutable(), ColumnUInt8::create()); + ColumnNullable::create(new_column_object1->assert_mutable(), ColumnUInt8::create()); size_t n = 1000; st = nested_iter->seek_to_ordinal(0); EXPECT_TRUE(st.ok()) << st.msg(); @@ -3724,7 +3724,7 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) { // fill with nullable ColumnVariant target MutableColumnPtr new_column_object12 = ColumnVariant::create(3, false); MutableColumnPtr null_object12 = ColumnNullable::create( - new_column_object12->assume_mutable(), ColumnUInt8::create()); + new_column_object12->assert_mutable(), ColumnUInt8::create()); st = nested_iter->seek_to_ordinal(0); EXPECT_TRUE(st.ok()) << st.msg(); st = nested_iter->next_batch(&n, null_object12, &has_null); @@ -3756,7 +3756,7 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter) { // fill with nullable ColumnVariant target MutableColumnPtr new_column_object2 = ColumnVariant::create(3, false); MutableColumnPtr null_object2 = - ColumnNullable::create(new_column_object2->assume_mutable(), ColumnUInt8::create()); + ColumnNullable::create(new_column_object2->assert_mutable(), ColumnUInt8::create()); size_t nrows = 1000; st = nested_iter2->seek_to_ordinal(0); EXPECT_TRUE(st.ok()) << st.msg(); @@ -3866,7 +3866,7 @@ TEST_F(VariantColumnWriterReaderTest, test_nested_iter_nullable) { // fill with nullable ColumnVariant target MutableColumnPtr new_column_object1 = ColumnVariant::create(3, false); MutableColumnPtr null_object = - ColumnNullable::create(new_column_object1->assume_mutable(), ColumnUInt8::create()); + ColumnNullable::create(new_column_object1->assert_mutable(), ColumnUInt8::create()); size_t nrows = 1000; st = nested_iter->seek_to_ordinal(0); EXPECT_TRUE(st.ok()) << st.msg(); diff --git a/be/test/storage/segment/variant_stats_calculator_test.cpp b/be/test/storage/segment/variant_stats_calculator_test.cpp index 66c3050370c6fd..1f5143ca422f8a 100644 --- a/be/test/storage/segment/variant_stats_calculator_test.cpp +++ b/be/test/storage/segment/variant_stats_calculator_test.cpp @@ -312,7 +312,7 @@ TEST_F(VariantStatsCalculatorTest, CalculateVariantStatsWithMultipleColumns) { std::make_shared(std::make_shared()), "sub1"}); auto map_col = create_map_column(); - map_col->assume_mutable()->insert_many_defaults(3); + map_col->assert_mutable()->insert_many_defaults(3); block.insert({std::move(map_col), std::make_shared(std::make_shared(), std::make_shared()), diff --git a/be/test/testutil/test_util.cpp b/be/test/testutil/test_util.cpp index 9615c6e9a107b2..bef28e075f6baa 100644 --- a/be/test/testutil/test_util.cpp +++ b/be/test/testutil/test_util.cpp @@ -189,8 +189,11 @@ void load_data_from_csv(const DataTypeSerDeSPtrs serders, MutableColumns& column << "serder size: " << serders.size() << " column size: " << columns.size(); ASSERT_EQ(serders.size(), idxes.size()) << "serder size: " << serders.size() << " idxes size: " << idxes.size(); - ASSERT_EQ(serders.size(), *idxes.end()) - << "serder size: " << serders.size() << " idxes size: " << *idxes.end(); + // Note: an earlier ASSERT here dereferenced idxes.end() (undefined + // behaviour on std::set). On Linux libstdc++ it happened to yield a value + // matching the previous check, so the assertion was effectively a + // duplicate of the size check above. On macOS libc++ it yields a + // different value and aborts the test. Drop the redundant UB assertion. std::ifstream file(file_path); if (!file) { throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "can not open the file: {} ", diff --git a/be/test/udf/python/python_env_test.cpp b/be/test/udf/python/python_env_test.cpp index 0ccbb63cf1d75e..a0e1fa2d883a85 100644 --- a/be/test/udf/python/python_env_test.cpp +++ b/be/test/udf/python/python_env_test.cpp @@ -53,7 +53,7 @@ class PythonEnvTest : public ::testing::Test { // which causes pclose() to get ECHILD because the kernel auto-reaps children. // We reset SIGCHLD to SIG_DFL for the duration of each test to mimic production // behaviour, and restore the original handler afterwards. - sighandler_t old_sigchld_ = SIG_DFL; + void (*old_sigchld_)(int) = SIG_DFL; void SetUp() override { test_dir_ = fs::temp_directory_path().string() + "/python_env_test_" + diff --git a/run-be-ut.sh b/run-be-ut.sh index 50b76f6d6f16bd..e427ab36df66ac 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -42,6 +42,27 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" export ROOT export DORIS_HOME="${ROOT}" +# On macOS, prefer Apple's Clang 17 (from Xcode Command Line Tools) over +# Homebrew's llvm@18/20. Apple Clang 17: +# - Does not have libc++ 18/20 breaking changes (atomic static_assert, compressed_pair) +# - Accepts template-template params with default args (P0522R0), which llvm@18 rejects +# - Does not have the structured-binding-in-OpenMP Clang@18 restriction +if [[ "$(uname -s)" == 'Darwin' && -x "/Library/Developer/CommandLineTools/usr/bin/clang++" ]]; then + export DORIS_CLANG_HOME="/Library/Developer/CommandLineTools/usr" +fi + +# On macOS, env.sh requires JDK-17. If JAVA_HOME is not set, auto-detect the +# Homebrew openjdk@17 installation. Use the real JDK home (Contents/Home) rather +# than the Homebrew prefix, so that find_libjvm.sh can locate libjvm.dylib and +# cmake can find the JNI headers. +if [[ "$(uname -s)" == 'Darwin' && -z "${JAVA_HOME}" ]]; then + _jdk_home="/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home" + if [[ -x "${_jdk_home}/bin/java" ]]; then + export JAVA_HOME="${_jdk_home}" + fi + unset _jdk_home +fi + . "${DORIS_HOME}/env.sh" # Check args @@ -218,7 +239,12 @@ if [[ -z "${USE_LIBCPP}" ]]; then fi if [[ -z "${USE_AVX2}" ]]; then - USE_AVX2='ON' + # AVX2 is x86-only; arm64 and other non-x86 architectures do not have it + if [[ "$(uname -m)" == "x86_64" ]]; then + USE_AVX2='ON' + else + USE_AVX2='OFF' + fi fi if [[ -z "${ARM_MARCH}" ]]; then @@ -248,6 +274,31 @@ else BUILD_TYPE="${CMAKE_BUILD_TYPE}" fi +OPENMP_CMAKE_VARS=() +if [[ "$(uname -s)" == 'Darwin' ]]; then + OPENMP_CMAKE_VARS=( + # ANN (faiss/openblas) has libc++ compatibility issues with llvm@20 on macOS. + # Disable it for local UT builds; CI runs on Linux where it compiles fine. + "-DDISABLE_ANN=ON" + # Clang@18 does not support capturing structured bindings in OpenMP parallel + # regions (variant_util.cpp uses this pattern). Disable OpenMP for macOS local + # UT builds: #pragma omp directives become no-ops, which is fine for testing. + "-DOpenMP_C_FLAGS=" + "-DOpenMP_CXX_FLAGS=" + "-DOpenMP_C_LIB_NAMES=none" + "-DOpenMP_CXX_LIB_NAMES=none" + "-DCMAKE_DISABLE_FIND_PACKAGE_OpenMP=ON" + # Apple's clang requires an explicit sysroot to find standard C headers + # (stdio.h etc.) in third-party C code (clucene, vp4d.c, …). + "-DCMAKE_OSX_SYSROOT=$(xcrun --show-sdk-path)" + # Apple's ar/ranlib use a 32-bit archive offset table that overflows when + # ASAN-instrumented .o files exceed ~4 GB total. Use LLVM's ar/ranlib + # (from Homebrew llvm@18) which supports the GNU 64-bit extended format. + "-DCMAKE_AR=/opt/homebrew/opt/llvm@18/bin/llvm-ar" + "-DCMAKE_RANLIB=/opt/homebrew/opt/llvm@18/bin/llvm-ranlib" + ) +fi + cd "${CMAKE_BUILD_DIR}" "${CMAKE_CMD}" -G "${GENERATOR}" \ -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \ @@ -270,6 +321,7 @@ cd "${CMAKE_BUILD_DIR}" -DDORIS_JAVA_HOME="${JAVA_HOME}" \ -DBUILD_AZURE="${BUILD_AZURE}" \ -DWITH_TDE_DIR="${WITH_TDE_DIR}" \ + "${OPENMP_CMAKE_VARS[@]}" \ "${DORIS_HOME}/be" "${BUILD_SYSTEM}" -j "${PARALLEL}"