update gcc to gcc 10 and support c++17 (#5394)

* update gcc to gcc 10 and support c++17 update brpc to 0.9.7 update boost to 1.73 remove third-party boost 1.54 for mysql * update cmake version * ignore jdk version * remove unused patch * avoid use SYS_getrandom call
apache · Mar 25, 2021 · ad67dd3 · ad67dd3
1 parent 23bd4a9
commit ad67dd3
Show file tree

Hide file tree

Showing 33 changed files with 183 additions and 371 deletions.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cmake_minimum_required(VERSION 3.12.0)
+cmake_minimum_required(VERSION 3.19.2)
 
 # set CMAKE_C_COMPILER, this must set before project command
 if (DEFINED ENV{DORIS_GCC_HOME})
@@ -130,14 +130,13 @@ endif()
 set(Boost_DEBUG FALSE)
 set(Boost_USE_MULTITHREADED ON)
 set(BOOST_ROOT ${THIRDPARTY_DIR})
+set(Boost_NO_BOOST_CMAKE OFF)
 
 if (NOT APPLE)
-    find_package(Boost 1.55.0 REQUIRED COMPONENTS thread regex filesystem system date_time program_options)
+    find_package(Boost 1.73.0 REQUIRED COMPONENTS regex system filesystem thread date_time program_options)
 else()
-    find_package(Boost 1.55.0 COMPONENTS thread regex filesystem system date_time program_options)
+    find_package(Boost 1.73.0 COMPONENTS thread regex system filesystem date_time program_options)
 endif()
-include_directories(${Boost_INCLUDE_DIRS})
-message(STATUS ${Boost_LIBRARIES})
 
 set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
 
@@ -313,10 +312,13 @@ check_function_exists(sched_getcpu HAVE_SCHED_GETCPU)
 #  -fno-omit-frame-pointers: Keep frame pointer for functions in register
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-pragmas -pthread")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-strict-aliasing -fno-omit-frame-pointer")
-set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++11 -D__STDC_FORMAT_MACROS")
+set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++17 -D__STDC_FORMAT_MACROS")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED")
+# https://github.com/boostorg/uuid/issues/92
+# We need to avoid using SYS_getrandom system calls
+set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1")
 if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86" OR "${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86_64")
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse4.2")
 endif()
@@ -344,7 +346,7 @@ set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -g -Wno-unused-local-typedefs")
 # Debug information is stored as dwarf2 to be as compatible as possible
 #   -Werror: compile warnings should be errors when using the toolchain compiler.
 # Only enable for debug builds because this is what we test in pre-commit tests.
-set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -Werror -ggdb3 -O0 -gdwarf-2")
+set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -ggdb3 -O0 -gdwarf-2")
 
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
@@ -449,7 +451,12 @@ set(DORIS_DEPENDENCIES
     librdkafka
     libs2
     snappy
-    ${Boost_LIBRARIES}
+    Boost::regex
+    Boost::system
+    Boost::filesystem
+    Boost::thread
+    Boost::date_time
+    Boost::program_options
     thrift
     thriftnb
     glog

diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-cmake_minimum_required(VERSION 2.6)
-
 # where to put generated libraries
 set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exec")
 

diff --git a/be/src/exec/decompressor.cpp b/be/src/exec/decompressor.cpp
@@ -63,7 +63,7 @@ std::string Decompressor::debug_info() {
 
 // Gzip
 GzipDecompressor::GzipDecompressor(bool is_deflate)
-        : Decompressor(_is_deflate ? CompressType::DEFLATE : CompressType::GZIP),
+        : Decompressor(is_deflate ? CompressType::DEFLATE : CompressType::GZIP),
           _is_deflate(is_deflate) {}
 
 GzipDecompressor::~GzipDecompressor() {

diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h
@@ -435,16 +435,21 @@ void ColumnValueRange<T>::convert_to_fixed_value() {
         return;
     }
 
+    // Incrementing boolean is denied in C++17, So we use int as bool type
+    using type = std::conditional_t<std::is_same<bool, T>::value, int, T>;
+    type low_value = _low_value;
+    type high_value = _high_value;
+
     if (_low_op == FILTER_LARGER) {
-        ++_low_value;
+        ++low_value;
     }
 
-    for (T v = _low_value; v < _high_value; ++v) {
+    for (auto v = low_value; v < high_value; ++v) {
         _fixed_values.insert(v);
     }
 
     if (_high_op == FILTER_LESS_OR_EQUAL) {
-        _fixed_values.insert(_high_value);
+        _fixed_values.insert(high_value);
     }
 }
 

diff --git a/be/src/exec/scan_node.cpp b/be/src/exec/scan_node.cpp
@@ -17,8 +17,6 @@
 
 #include "exec/scan_node.h"
 
-#include <boost/bind.hpp>
-
 namespace doris {
 
 const std::string ScanNode::_s_bytes_read_counter = "BytesRead";

diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
@@ -21,7 +21,7 @@
 #include <gperftools/malloc_extension.h>
 
 #include <boost/algorithm/string.hpp>
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 #include <sstream>
 
 #include "agent/utils.h"
@@ -336,7 +336,7 @@ void add_default_path_handlers(WebPageHandler* web_page_handler,
     web_page_handler->register_page("/logs", "Logs", logs_handler, false /* is_on_nav_bar */);
     web_page_handler->register_page("/varz", "Configs", config_handler, true /* is_on_nav_bar */);
     web_page_handler->register_page(
-            "/memz", "Memory", boost::bind<void>(&mem_usage_handler, process_mem_tracker, _1, _2),
+            "/memz", "Memory", boost::bind<void>(&mem_usage_handler, process_mem_tracker, boost::placeholders::_1, boost::placeholders::_2),
             true /* is_on_nav_bar */);
     web_page_handler->register_page("/mem_tracker", "MemTracker", mem_tracker_handler,
                                     true /* is_on_nav_bar */);
@@ -345,7 +345,7 @@ void add_default_path_handlers(WebPageHandler* web_page_handler,
     web_page_handler->register_page("/cpu", "CPU Profile", cpu_handler, true /* is_on_nav_bar */);
     register_thread_display_page(web_page_handler);
     web_page_handler->register_template_page("/tablets_page", "Tablets",
-                                             boost::bind<void>(&display_tablets_callback, _1, _2),
+                                             boost::bind<void>(&display_tablets_callback, boost::placeholders::_1, boost::placeholders::_2),
                                              true /* is_on_nav_bar */);
 }
 

diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp
@@ -147,7 +147,7 @@ Status EvHttpServer::_bind() {
         ss << "convert address failed, host=" << _host << ", port=" << _port;
         return Status::InternalError(ss.str());
     }
-    _server_fd = butil::tcp_listen(point, true);
+    _server_fd = butil::tcp_listen(point);
     if (_server_fd < 0) {
         char buf[64];
         std::stringstream ss;

diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
@@ -18,6 +18,7 @@
 #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H
 #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H
 
+#include <atomic>
 #include <memory>
 #include <mutex>
 #include <vector>

diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <atomic>
+
 #include "olap/rowset/rowset_id_generator.h"
 #include "util/spinlock.h"
 #include "util/uid_util.h"

diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
@@ -597,6 +597,7 @@ OLAPStatus StorageEngine::_start_trash_sweep(double* usage) {
 
     time_t now = time(nullptr); //获取UTC时间
     tm local_tm_now;
+    local_tm_now.tm_isdst = 0;
     if (localtime_r(&now, &local_tm_now) == nullptr) {
         LOG(WARNING) << "fail to localtime_r time. time=" << now;
         return OLAP_ERR_OS_ERROR;
@@ -721,6 +722,7 @@ OLAPStatus StorageEngine::_do_sweep(const string& scan_root, const time_t& local
             string dir_name = item->path().filename().string();
             string str_time = dir_name.substr(0, dir_name.find('.'));
             tm local_tm_create;
+            local_tm_create.tm_isdst = 0;
             if (strptime(str_time.c_str(), "%Y%m%d%H%M%S", &local_tm_create) == nullptr) {
                 LOG(WARNING) << "fail to strptime time. [time=" << str_time << "]";
                 res = OLAP_ERR_OS_ERROR;

diff --git a/be/src/olap/task/engine_batch_load_task.cpp b/be/src/olap/task/engine_batch_load_task.cpp
@@ -161,7 +161,7 @@ AgentStatus EngineBatchLoadTask::_get_tmp_file_dir(const string& root_path, stri
         boost::system::error_code error_code;
         boost::filesystem::create_directories(*download_path, error_code);
 
-        if (0 != error_code) {
+        if (error_code.failed()) {
             status = DORIS_ERROR;
             LOG(WARNING) << "create download dir failed.path: " << *download_path
                          << ", error code: " << error_code;

diff --git a/be/src/runtime/buffered_block_mgr2.cc b/be/src/runtime/buffered_block_mgr2.cc
@@ -781,7 +781,7 @@ Status BufferedBlockMgr2::write_unpinned_block(Block* block) {
         }
         disk_id %= _io_mgr->num_local_disks();
         DiskIoMgr::WriteRange::WriteDoneCallback callback =
-                bind(mem_fn(&BufferedBlockMgr2::write_complete), this, block, _1);
+                bind(mem_fn(&BufferedBlockMgr2::write_complete), this, block, boost::placeholders::_1);
         block->_write_range = _obj_pool.add(
                 new DiskIoMgr::WriteRange(tmp_file->path(), file_offset, disk_id, callback));
         block->_tmp_file = tmp_file;

diff --git a/be/src/runtime/buffered_tuple_stream2.cc b/be/src/runtime/buffered_tuple_stream2.cc
@@ -17,7 +17,7 @@
 
 #include "runtime/buffered_tuple_stream2.h"
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 
 #include "runtime/descriptors.h"
 #include "runtime/row_batch.h"

diff --git a/be/src/runtime/buffered_tuple_stream3.cc b/be/src/runtime/buffered_tuple_stream3.cc
@@ -17,7 +17,7 @@
 
 #include <gutil/strings/substitute.h>
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 
 #include "runtime/buffered_tuple_stream3.inline.h"
 #include "runtime/bufferpool/reservation_tracker.h"

diff --git a/be/src/runtime/bufferpool/buffer_allocator.cc b/be/src/runtime/bufferpool/buffer_allocator.cc
@@ -17,7 +17,7 @@
 
 #include "runtime/bufferpool/buffer_allocator.h"
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 #include <mutex>
 
 #include "common/atomic.h"
@@ -721,8 +721,7 @@ std::string BufferPool::FreeBufferArena::DebugString() {
            << " free buffers: " << lists.num_free_buffers.load()
            << " low water mark: " << lists.low_water_mark
            << " clean pages: " << lists.num_clean_pages.load() << " ";
-        lists.clean_pages.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, _1));
-
+        lists.clean_pages.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, boost::placeholders::_1));
         ss << "\n";
     }
     return ss.str();

diff --git a/be/src/runtime/bufferpool/buffer_pool.cc b/be/src/runtime/bufferpool/buffer_pool.cc
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 #include <limits>
 #include <sstream>
 
@@ -700,11 +700,11 @@ string BufferPool::Client::DebugString() {
        << " in_flight_write_bytes: " << in_flight_write_pages_.bytes()
        << " reservation: " << reservation_.DebugString();
     ss << "\n  " << pinned_pages_.size() << " pinned pages: ";
-    pinned_pages_.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, _1));
+    pinned_pages_.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, boost::placeholders::_1));
     ss << "\n  " << dirty_unpinned_pages_.size() << " dirty unpinned pages: ";
-    dirty_unpinned_pages_.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, _1));
+    dirty_unpinned_pages_.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, boost::placeholders::_1));
     ss << "\n  " << in_flight_write_pages_.size() << " in flight write pages: ";
-    in_flight_write_pages_.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, _1));
+    in_flight_write_pages_.iterate(boost::bind<bool>(Page::DebugStringCallback, &ss, boost::placeholders::_1));
     return ss.str();
 }
 

diff --git a/be/src/runtime/client_cache.h b/be/src/runtime/client_cache.h
@@ -18,7 +18,7 @@
 #ifndef DORIS_BE_RUNTIME_CLIENT_CACHE_H
 #define DORIS_BE_RUNTIME_CLIENT_CACHE_H
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 #include <boost/thread/mutex.hpp>
 #include <boost/unordered_map.hpp>
 #include <list>
@@ -191,12 +191,12 @@ class ClientCache {
 
     ClientCache() : _client_cache_helper() {
         _client_factory = boost::bind<ThriftClientImpl*>(boost::mem_fn(&ClientCache::make_client),
-                                                         this, _1, _2);
+                                                         this, boost::placeholders::_1, boost::placeholders::_2);
     }
 
     ClientCache(int max_cache_size) : _client_cache_helper(max_cache_size) {
         _client_factory = boost::bind<ThriftClientImpl*>(boost::mem_fn(&ClientCache::make_client),
-                                                         this, _1, _2);
+                                                         this, boost::placeholders::_1, boost::placeholders::_2);
     }
 
     // Close all clients connected to the supplied address, (e.g., in

diff --git a/be/src/runtime/data_stream_mgr.h b/be/src/runtime/data_stream_mgr.h
@@ -104,7 +104,7 @@ class DataStreamMgr {
     // less-than ordering for pair<TUniqueId, PlanNodeId>
     struct ComparisonOp {
         bool operator()(const std::pair<doris::TUniqueId, PlanNodeId>& a,
-                        const std::pair<doris::TUniqueId, PlanNodeId>& b) {
+                        const std::pair<doris::TUniqueId, PlanNodeId>& b) const {
             if (a.first.hi < b.first.hi) {
                 return true;
             } else if (a.first.hi > b.first.hi) {

diff --git a/be/src/runtime/data_stream_recvr.cc b/be/src/runtime/data_stream_recvr.cc
@@ -368,7 +368,7 @@ Status DataStreamRecvr::create_merger(const TupleRowComparator& less_than) {
 
     for (int i = 0; i < _sender_queues.size(); ++i) {
         child_input_batch_suppliers.emplace_back(
-                bind(mem_fn(&SenderQueue::get_batch), _sender_queues[i], _1));
+                bind(mem_fn(&SenderQueue::get_batch), _sender_queues[i], boost::placeholders::_1));
     }
     RETURN_IF_ERROR(_merger->prepare(child_input_batch_suppliers));
     return Status::OK();
@@ -403,13 +403,13 @@ Status DataStreamRecvr::create_parallel_merger(const TupleRowComparator& less_th
                 less_than, &_row_desc, _profile, mem_tracker, batch_size, false));
         vector<SortedRunMerger::RunBatchSupplier> input_batch_suppliers;
         for (int j = i; j < std::min((size_t)i + step, _sender_queues.size()); ++j) {
-            input_batch_suppliers.emplace_back(
-                    bind(mem_fn(&SenderQueue::get_batch), _sender_queues[j], _1));
+            input_batch_suppliers.emplace_back(bind(mem_fn(&SenderQueue::get_batch),
+                                                    _sender_queues[j], boost::placeholders::_1));
         }
         child_merger->prepare(input_batch_suppliers);
 
-        child_input_batch_suppliers.emplace_back(
-                bind(mem_fn(&SortedRunMerger::get_batch), child_merger.get(), _1));
+        child_input_batch_suppliers.emplace_back(bind(mem_fn(&SortedRunMerger::get_batch),
+                                                      child_merger.get(), boost::placeholders::_1));
         _child_mergers.emplace_back(std::move(child_merger));
     }
     RETURN_IF_ERROR(_merger->prepare(child_input_batch_suppliers, true));

diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
@@ -20,7 +20,7 @@
 #include <gperftools/profiler.h>
 #include <thrift/protocol/TDebugProtocol.h>
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 #include <memory>
 #include <sstream>
 
@@ -167,9 +167,9 @@ FragmentExecState::FragmentExecState(const TUniqueId& query_id,
           _fragment_instance_id(fragment_instance_id),
           _backend_num(backend_num),
           _exec_env(exec_env),
-          _executor(exec_env,
-                    boost::bind<void>(boost::mem_fn(&FragmentExecState::coordinator_callback), this,
-                                      _1, _2, _3)),
+          _executor(exec_env, boost::bind<void>(boost::mem_fn(&FragmentExecState::coordinator_callback),
+                                              this, boost::placeholders::_1, boost::placeholders::_2,
+                                              boost::placeholders::_3)),
           _timeout_second(-1),
           _fragments_ctx(fragments_ctx) {
     _start_time = DateTimeValue::local_time();
@@ -184,9 +184,9 @@ FragmentExecState::FragmentExecState(const TUniqueId& query_id,
           _backend_num(backend_num),
           _exec_env(exec_env),
           _coord_addr(coord_addr),
-          _executor(exec_env,
-                    boost::bind<void>(boost::mem_fn(&FragmentExecState::coordinator_callback), this,
-                                      _1, _2, _3)),
+          _executor(exec_env, boost::bind<void>(boost::mem_fn(&FragmentExecState::coordinator_callback),
+                                              this, boost::placeholders::_1, boost::placeholders::_2,
+                                              boost::placeholders::_3)),
           _timeout_second(-1) {
     _start_time = DateTimeValue::local_time();
 }
@@ -385,17 +385,17 @@ FragmentMgr::FragmentMgr(ExecEnv* exec_env)
     });
 
     auto s = Thread::create(
-                  "FragmentMgr", "cancel_timeout_plan_fragment",
-                  [this]() { this->cancel_worker(); }, &_cancel_thread);
+            "FragmentMgr", "cancel_timeout_plan_fragment", [this]() { this->cancel_worker(); },
+            &_cancel_thread);
     CHECK(s.ok()) << s.to_string();
 
     // TODO(zc): we need a better thread-pool
     // now one user can use all the thread pool, others have no resource.
     s = ThreadPoolBuilder("FragmentMgrThreadPool")
-            .set_min_threads(config::fragment_pool_thread_num_min)
-            .set_max_threads(config::fragment_pool_thread_num_max)
-            .set_max_queue_size(config::fragment_pool_queue_size)
-            .build(&_thread_pool);
+                .set_min_threads(config::fragment_pool_thread_num_min)
+                .set_max_threads(config::fragment_pool_thread_num_max)
+                .set_max_queue_size(config::fragment_pool_queue_size)
+                .build(&_thread_pool);
     CHECK(s.ok()) << s.to_string();
 }
 

diff --git a/be/src/runtime/result_buffer_mgr.cpp b/be/src/runtime/result_buffer_mgr.cpp
@@ -17,7 +17,7 @@
 
 #include "runtime/result_buffer_mgr.h"
 
-#include <boost/bind.hpp>
+#include <boost/bind/bind.hpp>
 
 #include "gen_cpp/PaloInternalService_types.h"
 #include "gen_cpp/types.pb.h"