[typo](docs)Fix Docs Error Urls

.github/actions/patches/action-sh-checker.patch

@@ -0,0 +1,13 @@
+diff --git a/entrypoint.sh b/entrypoint.sh
+index d3399e3..5c8ee7b 100755
+--- a/entrypoint.sh
++++ b/entrypoint.sh
+@@ -202,7 +202,7 @@ if ((CHECKBASHISMS_ENABLE == 1)); then
+ fi
+
+ if ((shellcheck_code != 0 || shfmt_code != 0)); then
+-	if [ "$GITHUB_EVENT_NAME" == "pull_request" ] && ((SH_CHECKER_COMMENT == 1)); then
++	if [[ "$GITHUB_EVENT_NAME" == "pull_request" || "$GITHUB_EVENT_NAME" == "pull_request_target" ]] && ((SH_CHECKER_COMMENT == 1)); then
+ 		_comment_on_github "$shellcheck_error" "$shfmt_error"
+ 	fi
+ fi

.github/workflows/build-extension.yml

@@ -99,6 +99,7 @@ jobs:
           rm -rf  website/i18n/zh-CN/docusaurus-plugin-content-docs-community/*
           mkdir website/i18n/zh-CN/docusaurus-plugin-content-docs-community/current
           cp -R docs/zh-CN/community/* website/i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
+          cp -R docs/sidebarsCommunity.json website/
           cp -R docs/sidebars.json website/
           cp -R docs/images website/static/
           sed '2,3d' website/versions.json > website/versions.json1

.github/workflows/shellcheck.yml

@@ -17,18 +17,32 @@
 
 name: ShellCheck
 
-on: [push, pull_request]
+on: [push, pull_request_target]
 
 jobs:
   shellcheck:
     name: ShellCheck 
     runs-on: ubuntu-latest
     steps:
       - name: Checkout ${{ github.ref }} ( ${{ github.sha }} )
+        if: ${{ github.event_name != 'pull_request_target' }}
         uses: actions/checkout@v3
         with:
           submodules: recursive
 
+      - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
+        if: ${{ github.event_name == 'pull_request_target' }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
+
+      - name: Patch
+        run: |
+          pushd .github/actions/action-sh-checker >/dev/null
+          git apply ../patches/action-sh-checker.patch
+          popd >/dev/null
+
       - name: Run ShellCheck
         uses: ./.github/actions/action-sh-checker
         env:

.licenserc.yaml

@@ -48,6 +48,13 @@ header:
     - "be/src/util/coding.h"
     - "be/src/util/condition_variable.cpp"
     - "be/src/util/condition_variable.h"
+    - "be/src/util/jsonb_document.h"
+    - "be/src/util/jsonb_error.h"
+    - "be/src/util/jsonb_parser.h"
+    - "be/src/util/jsonb_stream.h"
+    - "be/src/util/jsonb_updater.h"
+    - "be/src/util/jsonb_utils.h"
+    - "be/src/util/jsonb_writer.h"
     - "be/src/util/murmur_hash3.cpp"
     - "be/src/util/murmur_hash3.h"
     - "be/src/util/mustache/mustache.cc"

LICENSE.txt

@@ -747,3 +747,14 @@ The above copyright notice and this permission notice shall be included in all c
 
 THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 ----------------------------------------------------------------------------------
+
+be/src/util/jsonb*: BSD-style license
+
+Copyright (c) 2014, Facebook, Inc.
+All rights reserved.
+
+This source code is licensed under the BSD-style license found in the
+LICENSE file in the root directory of this source tree. An additional grant
+of patent rights can be found in the PATENTS file in the same directory.
+
+----------------------------------------------------------------------------------

README.md

@@ -27,7 +27,7 @@ under the License.
 [![Jenkins Vec](https://img.shields.io/jenkins/tests?compact_message&jobUrl=https://ci-builds.apache.org/job/Doris/job/doris_daily_disable_vectorized&label=OriginEngine)](https://ci-builds.apache.org/job/Doris/job/doris_daily_disable_vectorized)
 [![Jenkins Ori](https://img.shields.io/jenkins/tests?compact_message&jobUrl=https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized&label=VectorizedEngine)](https://ci-builds.apache.org/job/Doris/job/doris_daily_enable_vectorized)
 [![Total Lines](https://tokei.rs/b1/github/apache/doris?category=lines)](https://github.com/apache/doris)
-[![Join the Doris Community at Slack](https://img.shields.io/badge/chat-slack-brightgreen)](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-11jb8gesh-7IukzSrdea6mqoG0HB4gZg)
+[![Join the Doris Community at Slack](https://img.shields.io/badge/chat-slack-brightgreen)](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-1h153f1ar-sTJB_QahY1SHvZdtPFoIOQ)
 [![Join the chat at https://gitter.im/apache-doris/Lobby](https://badges.gitter.im/apache-doris/Lobby.svg)](https://gitter.im/apache-doris/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![EN doc](https://img.shields.io/badge/Docs-English-blue.svg)](https://doris.apache.org/docs/get-starting/)
 [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](https://doris.apache.org/zh-CN/docs/get-starting/)
@@ -222,7 +222,7 @@ Contact us through the following mailing list.
 
 * Apache Doris Official Website - [https://doris.apache.org](https://doris.apache.org)
 * Developer Mailing list - <dev@doris.apache.org>. Mail to <dev-subscribe@doris.apache.org>, follow the reply to subscribe the mail list.
-* Slack channel - [Join the Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-18u6vjopj-Th15vTVfmCzVfhhL5rz26A)
+* Slack channel - [Join the Slack](https://join.slack.com/t/apachedoriscommunity/shared_invite/zt-1h153f1ar-sTJB_QahY1SHvZdtPFoIOQ)
 * Twitter - [Follow @doris_apache](https://twitter.com/doris_apache)
 
 

be/CMakeLists.txt

@@ -223,7 +223,10 @@ add_library(leveldb STATIC IMPORTED)
 set_target_properties(leveldb PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libleveldb.a)
 
 add_library(jemalloc STATIC IMPORTED)
-set_target_properties(jemalloc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libjemalloc.a)
+set_target_properties(jemalloc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libjemalloc_doris.a)
+
+add_library(jemalloc_arrow STATIC IMPORTED)
+set_target_properties(jemalloc_arrow PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libjemalloc.a)
 
 add_library(brotlicommon STATIC IMPORTED)
 set_target_properties(brotlicommon PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libbrotlicommon.a)
@@ -410,14 +413,18 @@ check_function_exists(sched_getcpu HAVE_SCHED_GETCPU)
 #  -pthread: enable multithreaded malloc
 #  -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG: enable nanosecond precision for boost
 #  -fno-omit-frame-pointers: Keep frame pointer for functions in register
-set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -frecord-gcc-switches -Wall -Wno-sign-compare -pthread -Werror")
+set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -pthread -Werror")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fstrict-aliasing -fno-omit-frame-pointer")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++17 -D__STDC_FORMAT_MACROS")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG")
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED")
 # Enable the cpu and heap profile of brpc
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBRPC_ENABLE_CPU_PROFILER")
 
+if (RECORD_COMPILER_SWITCHES)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -frecord-gcc-switches")
+endif()
+
 function(TRY_TO_CHANGE_LINKER LINKER_COMMAND LINKER_NAME)
     if (CUSTUM_LINKER_COMMAND STREQUAL "ld")
         execute_process(COMMAND ${CMAKE_C_COMPILER} -fuse-ld=${LINKER_COMMAND} -Wl,--version ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
@@ -432,6 +439,7 @@ endfunction()
 set(CUSTUM_LINKER_COMMAND "ld")
 # TODO: mold will link fail on thirdparty brpc now, waiting for investigation.
 # TRY_TO_CHANGE_LINKER("mold" "mold")
+
 TRY_TO_CHANGE_LINKER("lld" "LLD")
 TRY_TO_CHANGE_LINKER("gold" "GNU gold")
 if (NOT CUSTUM_LINKER_COMMAND STREQUAL "ld")
@@ -452,7 +460,7 @@ if (COMPILER_GCC)
 endif ()
 
 if (COMPILER_CLANG)
-    add_compile_options (-fcolor-diagnostics)
+    add_compile_options(-fcolor-diagnostics -fno-limit-debug-info)
     if(MAKE_TEST STREQUAL "OFF")
         add_compile_options(-Qunused-arguments)
     endif()
@@ -676,6 +684,7 @@ set(COMMON_THIRDPARTY
     roaring
     fmt
     jemalloc
+    jemalloc_arrow
     brotlicommon
     brotlidec
     brotlienc

be/src/agent/heartbeat_server.cpp

@@ -41,6 +41,15 @@ using apache::thrift::TProcessor;
 
 namespace doris {
 
+// When we have some breaking change for execute engine, we should update be_exec_version.
+// 0: not contain be_exec_version.
+// 1: remove ColumnString's terminating zero.
+const int HeartbeatServer::max_be_exec_version = 1;
+const int HeartbeatServer::min_be_exec_version = 0;
+
+// For support rolling upgrade, we send data as second newest version.
+int HeartbeatServer::be_exec_version = max_be_exec_version - 1;
+
 HeartbeatServer::HeartbeatServer(TMasterInfo* master_info)
         : _master_info(master_info), _fe_epoch(0) {
     _olap_engine = StorageEngine::instance();
@@ -161,6 +170,13 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) {
         _master_info->__set_backend_id(master_info.backend_id);
     }
 
+    if (master_info.__isset.be_exec_version && check_be_exec_version(master_info.be_exec_version) &&
+        be_exec_version != master_info.be_exec_version) {
+        LOG(INFO) << fmt::format("be_exec_version changed from {} to {}", be_exec_version,
+                                 master_info.be_exec_version);
+        be_exec_version = master_info.be_exec_version;
+    }
+
     if (need_report) {
         LOG(INFO) << "Master FE is changed or restarted. report tablet and disk info immediately";
         _olap_engine->notify_listeners();

be/src/agent/heartbeat_server.h

@@ -34,7 +34,7 @@ class ThriftServer;
 class HeartbeatServer : public HeartbeatServiceIf {
 public:
     explicit HeartbeatServer(TMasterInfo* master_info);
-    virtual ~HeartbeatServer() {};
+    ~HeartbeatServer() override = default;
 
     virtual void init_cluster_id();
 
@@ -45,7 +45,24 @@ class HeartbeatServer : public HeartbeatServiceIf {
     //
     // Output parameters:
     // * heartbeat_result: The result of heartbeat set
-    virtual void heartbeat(THeartbeatResult& heartbeat_result, const TMasterInfo& master_info);
+    void heartbeat(THeartbeatResult& heartbeat_result, const TMasterInfo& master_info) override;
+
+    static const int max_be_exec_version;
+    static const int min_be_exec_version;
+    static int be_exec_version;
+
+    static bool check_be_exec_version(int be_exec_version) {
+        if (be_exec_version > max_be_exec_version || be_exec_version < min_be_exec_version) {
+            LOG(WARNING) << fmt::format(
+                    "Received be_exec_version is not supported, be_exec_version={}, "
+                    "min_be_exec_version={}, max_be_exec_version={}, maybe due to FE version not "
+                    "match "
+                    "with BE.",
+                    be_exec_version, min_be_exec_version, max_be_exec_version);
+            return false;
+        }
+        return true;
+    }
 
 private:
     Status _heartbeat(const TMasterInfo& master_info);

be/src/agent/task_worker_pool.cpp

@@ -722,14 +722,12 @@ void TaskWorkerPool::_publish_version_worker_thread_callback() {
                     .error(status);
             finish_task_request.__set_error_tablet_ids(error_tablet_ids);
         } else {
-            int submit_tablets = 0;
             if (config::enable_quick_compaction && config::quick_compaction_batch_size > 0) {
                 for (int i = 0; i < succ_tablet_ids.size(); i++) {
                     TabletSharedPtr tablet =
                             StorageEngine::instance()->tablet_manager()->get_tablet(
                                     succ_tablet_ids[i]);
                     if (tablet != nullptr) {
-                        submit_tablets++;
                         tablet->publised_count++;
                         if (tablet->publised_count % config::quick_compaction_batch_size == 0) {
                             StorageEngine::instance()->submit_quick_compaction_task(tablet);
@@ -856,6 +854,14 @@ void TaskWorkerPool::_update_tablet_meta_worker_thread_callback() {
                     if (tablet_meta_info.storage_policy.empty()) {
                         tablet->tablet_meta()->mutable_tablet_schema()->set_is_in_memory(
                                 tablet_meta_info.is_in_memory);
+                        // The field is_in_memory should not be in the tablet_schema.
+                        // it should be in the tablet_meta.
+                        for (auto rowset_meta : tablet->tablet_meta()->all_mutable_rs_metas()) {
+                            rowset_meta->tablet_schema()->set_is_in_memory(
+                                    tablet_meta_info.is_in_memory);
+                        }
+                        tablet->get_max_version_schema(wrlock)->set_is_in_memory(
+                                tablet_meta_info.is_in_memory);
                     } else {
                         LOG(INFO) << "set tablet cooldown resource "
                                   << tablet_meta_info.storage_policy;

be/src/common/config.h

@@ -257,9 +257,6 @@ CONF_Bool(enable_vectorized_compaction, "true");
 // whether enable vectorized schema change/material-view/rollup task.
 CONF_Bool(enable_vectorized_alter_table, "true");
 
-// serialize data version
-CONF_mInt32(block_data_version, "-1");
-
 // check the configuration of auto compaction in seconds when auto compaction disabled
 CONF_mInt32(check_auto_compaction_interval_seconds, "5");
 
@@ -520,6 +517,13 @@ CONF_mInt64(memtable_max_buffer_size, "419430400");
 CONF_Int64(load_process_max_memory_limit_bytes, "107374182400"); // 100GB
 CONF_Int32(load_process_max_memory_limit_percent, "50");         // 50%
 
+// If the memory consumption of load jobs exceed load_process_max_memory_limit,
+// all load jobs will hang there to wait for memtable flush. We should have a
+// soft limit which can trigger the memtable flush for the load channel who
+// consumes lagest memory size before we reach the hard limit. The soft limit
+// might avoid all load jobs hang at the same time.
+CONF_Int32(load_process_soft_mem_limit_percent, "50");
+
 // result buffer cancelled time (unit: second)
 CONF_mInt32(result_buffer_cancelled_interval_time, "300");
 
@@ -581,7 +585,7 @@ CONF_mInt32(storage_flood_stage_usage_percent, "90"); // 90%
 // The min bytes that should be left of a data dir
 CONF_mInt64(storage_flood_stage_left_capacity_bytes, "1073741824"); // 1GB
 // number of thread for flushing memtable per store
-CONF_Int32(flush_thread_num_per_store, "2");
+CONF_Int32(flush_thread_num_per_store, "6");
 // number of thread for flushing memtable per store, for high priority load task
 CONF_Int32(high_priority_flush_thread_num_per_store, "1");
 
@@ -725,7 +729,7 @@ CONF_mInt32(max_segment_num_per_rowset, "200");
 // The connection timeout when connecting to external table such as odbc table.
 CONF_mInt32(external_table_connect_timeout_sec, "30");
 
-// The capacity of lur cache in segment loader.
+// The capacity of lru cache in segment loader.
 // Althought it is called "segment cache", but it caches segments in rowset granularity.
 // So the value of this config should corresponding to the number of rowsets on this BE.
 CONF_mInt32(segment_cache_capacity, "1000000");
@@ -804,15 +808,23 @@ CONF_mInt32(string_type_length_soft_limit_bytes, "1048576");
 CONF_Validator(string_type_length_soft_limit_bytes,
                [](const int config) -> bool { return config > 0 && config <= 2147483643; });
 
+CONF_mInt32(jsonb_type_length_soft_limit_bytes, "1048576");
+
+CONF_Validator(jsonb_type_length_soft_limit_bytes,
+               [](const int config) -> bool { return config > 0 && config <= 2147483643; });
+
 // used for olap scanner to save memory, when the size of unused_object_pool
 // is greater than object_pool_buffer_size, release the object in the unused_object_pool.
 CONF_Int32(object_pool_buffer_size, "100");
 
 // ParquetReaderWrap prefetch buffer size
 CONF_Int32(parquet_reader_max_buffer_size, "50");
-CONF_Bool(parquet_predicate_push_down, "true");
-CONF_Int32(parquet_header_max_size, "8388608");
-CONF_Bool(parquet_reader_using_internal, "false");
+// Max size of parquet page header in bytes
+CONF_mInt32(parquet_header_max_size_mb, "1");
+// Max buffer size for parquet row group
+CONF_mInt32(parquet_rowgroup_max_buffer_mb, "128");
+// Max buffer size for parquet chunk column
+CONF_mInt32(parquet_column_max_buffer_mb, "8");
 
 // When the rows number reached this limit, will check the filter rate the of bloomfilter
 // if it is lower than a specific threshold, the predicate will be disabled.
@@ -870,7 +882,10 @@ CONF_mInt64(nodechannel_pending_queue_max_bytes, "67108864");
 CONF_mInt32(max_fragment_start_wait_time_seconds, "30");
 
 // Temp config. True to use new file scan node to do load job. Will remove after fully test.
-CONF_Bool(enable_new_load_scan_node, "false");
+CONF_mBool(enable_new_load_scan_node, "false");
+
+// Temp config. True to use new file scanner. Will remove after fully test.
+CONF_mBool(enable_new_file_scanner, "false");
 
 #ifdef BE_TEST
 // test s3

be/src/common/daemon.cpp

@@ -68,6 +68,7 @@ namespace doris {
 bool k_doris_exit = false;
 
 void Daemon::tcmalloc_gc_thread() {
+    // TODO All cache GC wish to be supported
     while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(10))) {
         size_t used_size = 0;
         size_t free_size = 0;
@@ -76,6 +77,8 @@ void Daemon::tcmalloc_gc_thread() {
                                                         &used_size);
         MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes", &free_size);
         size_t alloc_size = used_size + free_size;
+        LOG(INFO) << "tcmalloc.pageheap_free_bytes " << free_size
+                  << ", generic.current_allocated_bytes " << used_size;
 
         if (alloc_size > config::tc_use_memory_min) {
             size_t max_free_size = alloc_size * config::tc_free_memory_rate / 100;