[Fix] Fix clang-18 and libc++ compile problems

be/CMakeLists.txt

@@ -155,6 +155,7 @@ set(BOOST_VERSION "1.81.0")
 
 if (NOT APPLE)
     find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system date_time)
+    find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system container)
 else()
     find_package(Boost ${BOOST_VERSION} COMPONENTS system date_time)
     find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
@@ -288,12 +289,11 @@ if (COMPILER_CLANG)
                         -Wno-implicit-float-conversion
                         -Wno-implicit-int-conversion
                         -Wno-sign-conversion
+                        -Wno-missing-field-initializers
+                        -Wno-unused-const-variable
                         -Wno-shorten-64-to-32)
     if (USE_LIBCPP)
         add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
-        if (NOT OS_MACOSX)
-            add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-lstdc++>)
-        endif()
         add_definitions(-DUSE_LIBCPP)
     endif()
 endif ()
@@ -494,6 +494,7 @@ find_package(absl)
 # add it here first.
 set(COMMON_THIRDPARTY
     Boost::date_time
+    Boost::container
     ${COMMON_THIRDPARTY}
 )
 
@@ -536,7 +537,6 @@ endif()
 if (OS_MACOSX)
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}
-        Boost::container
         bfd
         iberty
         intl
@@ -580,9 +580,11 @@ if (NOT OS_MACOSX)
         ${DORIS_DEPENDENCIES}
         -static-libstdc++
         -static-libgcc
-        -lstdc++fs
         -lresolv
     )
+    if (NOT (USE_LIBCPP AND COMPILER_CLANG))
+        set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs)
+    endif()
 else()
     set(DORIS_LINK_LIBS
         ${DORIS_LINK_LIBS}

be/src/cloud/cloud_tablet_hotspot.cpp

@@ -89,20 +89,20 @@ void TabletHotspot::get_top_n_hot_partition(std::vector<THotTableMessage>* hot_t
                 hot_partition.qpd = std::max(hot_partition.qpd, counter->qpd());
                 hot_partition.qpw = std::max(hot_partition.qpw, counter->qpw());
                 hot_partition.last_access_time =
-                        std::max(hot_partition.last_access_time,
-                                 std::chrono::duration_cast<std::chrono::seconds>(
-                                         counter->last_access_time.time_since_epoch())
-                                         .count());
+                        std::max<int64_t>(hot_partition.last_access_time,
+                                          std::chrono::duration_cast<std::chrono::seconds>(
+                                                  counter->last_access_time.time_since_epoch())
+                                                  .count());
             } else if (counter->qpw() != 0) {
                 auto& hot_partition = week_hot_partitions[std::make_pair(
                         counter->table_id, counter->index_id)][counter->partition_id];
                 hot_partition.qpd = 0;
                 hot_partition.qpw = std::max(hot_partition.qpw, counter->qpw());
                 hot_partition.last_access_time =
-                        std::max(hot_partition.last_access_time,
-                                 std::chrono::duration_cast<std::chrono::seconds>(
-                                         counter->last_access_time.time_since_epoch())
-                                         .count());
+                        std::max<int64_t>(hot_partition.last_access_time,
+                                          std::chrono::duration_cast<std::chrono::seconds>(
+                                                  counter->last_access_time.time_since_epoch())
+                                                  .count());
             }
         }
     });

be/src/io/cache/block_file_cache.cpp

@@ -151,7 +151,7 @@ void BlockFileCache::remove_query_context(const TUniqueId& query_id) {
     std::lock_guard cache_lock(_mutex);
     const auto& query_iter = _query_map.find(query_id);
 
-    if (query_iter != _query_map.end() && query_iter->second.unique()) {
+    if (query_iter != _query_map.end() && query_iter->second.use_count() <= 1) {
         _query_map.erase(query_iter);
     }
 }

be/src/io/cache/block_file_cache.h

@@ -289,7 +289,7 @@ class BlockFileCache {
         /// Pointer to file block is always hold by the cache itself.
         /// Apart from pointer in cache, it can be hold by cache users, when they call
         /// getorSet(), but cache users always hold it via FileBlocksHolder.
-        bool releasable() const { return file_block.unique(); }
+        bool releasable() const { return file_block.use_count() <= 1; }
 
         size_t size() const { return file_block->_block_range.size(); }
 

be/src/util/s3_rate_limiter.cpp

@@ -24,12 +24,6 @@
 #include "util/s3_util.h"
 #include "util/spinlock.h"
 #include "util/time.h"
-#if defined(__APPLE__)
-#include <ctime>
-#define CURRENT_TIME std::chrono::system_clock::now()
-#else
-#define CURRENT_TIME std::chrono::high_resolution_clock::now()
-#endif
 
 namespace doris {
 // Just 10^6.
@@ -57,7 +51,8 @@ std::pair<size_t, double> S3RateLimiter::_update_remain_token(
 
 int64_t S3RateLimiter::add(size_t amount) {
     // Values obtained under lock to be checked after release
-    auto [count_value, tokens_value] = _update_remain_token(CURRENT_TIME, amount);
+    auto [count_value, tokens_value] =
+            _update_remain_token(std::chrono::system_clock::now(), amount);
 
     if (_limit && count_value > _limit) {
         // CK would throw exception
@@ -126,4 +121,4 @@ Status S3RateLimiterHolder::reset(size_t max_speed, size_t max_burst, size_t lim
     }
     return Status::OK();
 }
-} // namespace doris
+} // namespace doris

be/test/util/threadpool_test.cpp

@@ -332,6 +332,8 @@ TEST_F(ThreadPoolTest, TestDeadlocks) {
     const char* death_msg = "doris::ThreadPool::check_not_pool_thread_unlocked()";
 #elif defined(__APPLE__)
     const char* death_msg = "pthread_start";
+#elif defined(__clang__) && defined(USE_LIBCPP)
+    const char* death_msg = "doris::ThreadPool::check_not_pool_thread_unlocked()";
 #else
     const char* death_msg =
             "_ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_"

cloud/CMakeLists.txt

@@ -164,7 +164,7 @@ if (NOT CUSTUM_LINKER_COMMAND STREQUAL "ld")
 endif()
 
 if (USE_LIBCPP AND COMPILER_CLANG)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++ -lstdc++")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
     add_definitions(-DUSE_LIBCPP)
 endif()
 
@@ -335,11 +335,13 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
     ${DORIS_DEPENDENCIES}
     -static-libstdc++
     -static-libgcc
-    -lstdc++fs
     -lresolv
     -L${DORIS_JAVA_HOME}/lib/server
     -ljvm
 )
+if (NOT (USE_LIBCPP AND COMPILER_CLANG))
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs)
+endif()
 
 if (USE_JEMALLOC)
     set(MALLOCLIB jemalloc)

cloud/src/common/encryption_util.cpp

@@ -635,8 +635,8 @@ static int generate_random_root_key(TxnKv* txn_kv, KmsClient* kms_client, std::s
 
     // 3. otherwise, generate a random data key in memory
     std::mt19937 rnd(time(nullptr));
-    std::uniform_int_distribution<char> dist(std::numeric_limits<char>::min(),
-                                             std::numeric_limits<char>::max());
+    std::uniform_int_distribution<short> dist(std::numeric_limits<char>::min(),
+                                              std::numeric_limits<char>::max());
     std::string root_key_plaintext(32, '0');
     for (char& i : root_key_plaintext) {
         i = (char)dist(rnd);

cloud/src/rate-limiter/s3_rate_limiter.cpp

@@ -49,7 +49,7 @@ std::pair<size_t, double> S3RateLimiter::_update_remain_token(
 int64_t S3RateLimiter::add(size_t amount) {
     // Values obtained under lock to be checked after release
     auto [count_value, tokens_value] =
-            _update_remain_token(std::chrono::high_resolution_clock::now(), amount);
+            _update_remain_token(std::chrono::system_clock::now(), amount);
 
     if (_limit && count_value > _limit) {
         // CK would throw exception
@@ -111,4 +111,4 @@ S3RateLimitType string_to_s3_rate_limit_type(std::string_view value) {
     }
     return S3RateLimitType::UNKNOWN;
 }
-} // namespace doris::cloud
+} // namespace doris::cloud

cloud/src/recycler/checker.cpp

@@ -126,8 +126,9 @@ int Checker::start() {
             long enqueue_time_s = 0;
             {
                 std::unique_lock lock(mtx_);
-                pending_instance_cond_.wait(
-                        lock, [&]() { return !pending_instance_queue_.empty() || stopped(); });
+                pending_instance_cond_.wait(lock, [&]() -> bool {
+                    return !pending_instance_queue_.empty() || stopped();
+                });
                 if (stopped()) {
                     return;
                 }

cloud/src/recycler/checker.h

@@ -17,6 +17,9 @@
 
 #pragma once
 
+#if defined(USE_LIBCPP) && _LIBCPP_ABI_VERSION <= 1
+#define _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE
+#endif
 #include <atomic>
 #include <condition_variable>
 #include <deque>

cloud/src/recycler/s3_accessor.cpp

@@ -511,7 +511,11 @@ int S3Accessor::check_bucket_versioning() {
 
 int GcsAccessor::delete_objects(const std::vector<std::string>& relative_paths) {
     std::vector<int> delete_rets(relative_paths.size());
+#ifdef USE_LIBCPP
+    std::transform(relative_paths.begin(), relative_paths.end(),
+#else
     std::transform(std::execution::par, relative_paths.begin(), relative_paths.end(),
+#endif
                    delete_rets.begin(),
                    [this](const std::string& path) { return delete_object(path); });
     int ret = 0;

cloud/test/codec_test.cpp

@@ -35,8 +35,8 @@ TEST(CodecTest, StringCodecTest) {
     std::mt19937 gen(std::random_device("/dev/urandom")());
     const int max_len = (2 << 16) + 10086;
     std::uniform_int_distribution<int> rd_len(0, max_len);
-    std::uniform_int_distribution<char> rd_char(std::numeric_limits<char>::min(),
-                                                std::numeric_limits<char>::max());
+    std::uniform_int_distribution<short> rd_char(std::numeric_limits<char>::min(),
+                                                 std::numeric_limits<char>::max());
 
     int ret = -1;
 

run-cloud-ut.sh

@@ -159,6 +159,14 @@ if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then
     GLIBC_COMPATIBILITY=ON
 fi
 
+if [[ -z "${USE_LIBCPP}" ]]; then
+    if [[ "$(uname -s)" != 'Darwin' ]]; then
+        USE_LIBCPP='OFF'
+    else
+        USE_LIBCPP='ON'
+    fi
+fi
+
 if [[ -z "${USE_DWARF}" ]]; then
     USE_DWARF=OFF
 fi
@@ -175,6 +183,7 @@ find . -name "*.gcda" -exec rm {} \;
     -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
     -DMAKE_TEST=ON \
     -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
+    -DUSE_LIBCPP="${USE_LIBCPP}" \
     -DUSE_DWARF="${USE_DWARF}" \
     -DUSE_MEM_TRACKER=ON \
     -DUSE_JEMALLOC=OFF \
@@ -204,6 +213,8 @@ echo "**********************************"
 echo "   Running MetaService Unit Test  "
 echo "**********************************"
 
+export ASAN_OPTIONS=detect_container_overflow=0
+
 # test binary output dir
 cd test
 # FILTER: binary_name:gtest_filter

thirdparty/build-thirdparty.sh

@@ -627,7 +627,7 @@ build_bzip() {
     check_if_source_exist "${BZIP_SOURCE}"
     cd "${TP_SOURCE_DIR}/${BZIP_SOURCE}"
 
-    make -j "${PARALLEL}" install PREFIX="${TP_INSTALL_DIR}"
+    make -j "${PARALLEL}" install PREFIX="${TP_INSTALL_DIR}" CFLAGS="-fPIC"
 }
 
 # lzo2
@@ -1803,7 +1803,7 @@ build_libuuid() {
     check_if_source_exist "${LIBUUID_SOURCE}"
     cd "${TP_SOURCE_DIR}/${LIBUUID_SOURCE}"
     CC=gcc ./configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static
-    make -j "${PARALLEL}"
+    make -j "${PARALLEL}" CFLAGS="-fPIC"
     make install
 }