Merge branch 'master' into fix-async-insert-dedup-for-merging-algorithms

SECURITY.md

@@ -13,6 +13,7 @@ The following versions of ClickHouse server are currently being supported with s
 
 | Version | Supported |
 |:-|:-|
+| 23.6 | ✔️ |
 | 23.5 | ✔️ |
 | 23.4 | ✔️ |
 | 23.3 | ✔️ |

cmake/autogenerated_versions.txt

@@ -2,11 +2,11 @@
 
 # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54475)
+SET(VERSION_REVISION 54476)
 SET(VERSION_MAJOR 23)
-SET(VERSION_MINOR 6)
+SET(VERSION_MINOR 7)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 2fec796e73efda10a538a03af3205ce8ffa1b2de)
-SET(VERSION_DESCRIBE v23.6.1.1-testing)
-SET(VERSION_STRING 23.6.1.1)
+SET(VERSION_GITHASH d1c7e13d08868cb04d3562dcced704dd577cb1df)
+SET(VERSION_DESCRIBE v23.7.1.1-testing)
+SET(VERSION_STRING 23.7.1.1)
 # end of autochange

docker/keeper/Dockerfile

@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
     esac
 
 ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release"
-ARG VERSION="23.5.4.25"
+ARG VERSION="23.6.1.1524"
 ARG PACKAGES="clickhouse-keeper"
 
 # user/group precreated explicitly with fixed uid/gid on purpose.

docker/server/Dockerfile.alpine

@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="23.5.4.25"
+ARG VERSION="23.6.1.1524"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # user/group precreated explicitly with fixed uid/gid on purpose.

docker/server/Dockerfile.ubuntu

@@ -23,7 +23,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="23.5.4.25"
+ARG VERSION="23.6.1.1524"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 # set non-empty deb_location_url url to create a docker image

docker/test/stateless/run.sh

@@ -18,6 +18,9 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
 # shellcheck disable=SC1091
 source /usr/share/clickhouse-test/ci/attach_gdb.lib || true  # FIXME: to not break old builds, clean on 2023-09-01
 
+# shellcheck disable=SC1091
+source /usr/share/clickhouse-test/ci/utils.lib
+
 # install test configs
 /usr/share/clickhouse-test/config/install.sh
 
@@ -90,30 +93,6 @@ sleep 5
 
 attach_gdb_to_clickhouse || true  # FIXME: to not break old builds, clean on 2023-09-01
 
-function run_with_retry()
-{
-    set +e
-
-    local total_retries="$1"
-    shift
-
-    local retry=0
-
-    until [ "$retry" -ge "$total_retries" ]
-    do
-        if "$@"; then
-            set -e
-            return
-        else
-            retry=$((retry + 1))
-            sleep 3
-        fi
-    done
-
-    echo "Command '$*' failed after $total_retries retries, exiting"
-    exit 1
-}
-
 function run_tests()
 {
     set -x
@@ -161,9 +140,7 @@ function run_tests()
 
     ADDITIONAL_OPTIONS+=('--report-logs-stats')
 
-    clickhouse-test "00001_select_1" > /dev/null ||:
-
-    run_with_retry 5 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
+    run_with_retry 10 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
 
     set +e
     clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \

docs/en/operations/server-configuration-parameters/settings.md

@@ -2120,7 +2120,13 @@ This section contains the following parameters:
 - `operation_timeout_ms` — Maximum timeout for one operation in milliseconds.
 - `root` — The [znode](http://zookeeper.apache.org/doc/r3.5.5/zookeeperOver.html#Nodes+and+ephemeral+nodes) that is used as the root for znodes used by the ClickHouse server. Optional.
 - `identity` — User and password, that can be required by ZooKeeper to give access to requested znodes. Optional.
-
+- zookeeper_load_balancing - Specifies the algorithm of ZooKeeper node selection.
+  * random - randomly selects one of ZooKeeper nodes.
+  * in_order - selects the first ZooKeeper node, if it's not available then the second, and so on.
+  * nearest_hostname - selects a ZooKeeper node with a hostname that is most similar to the server’s hostname.
+  * first_or_random - selects the first ZooKeeper node, if it's not available then randomly selects one of remaining ZooKeeper nodes.
+  * round_robin - selects the first ZooKeeper node, if reconnection happens selects the next.
+
 **Example configuration**
 
 ``` xml
@@ -2139,6 +2145,8 @@ This section contains the following parameters:
     <root>/path/to/zookeeper/node</root>
     <!-- Optional. Zookeeper digest ACL string. -->
     <identity>user:password</identity>
+    <!--<zookeeper_load_balancing>random / in_order / nearest_hostname / first_or_random / round_robin</zookeeper_load_balancing>-->
+    <zookeeper_load_balancing>random</zookeeper_load_balancing>
 </zookeeper>
 ```
 

docs/en/operations/storing-data.md

@@ -184,13 +184,15 @@ These settings should be defined in the disk configuration section.
 
 - `enable_filesystem_query_cache_limit` - allow to limit the size of cache which is downloaded within each query (depends on user setting `max_query_cache_size`). Default: `false`.
 
-- `enable_cache_hits_threshold` - number which defines how many times some data needs to be read before it will be cached. Default: `0`, e.g. the data is cached at the first attempt to read it.
+- `enable_cache_hits_threshold` - number which defines how many times some data needs to be read before it will be cached. Default: `false`. This threshold can be defined by `cache_hits_threshold`. Default: `0`, e.g. the data is cached at the first attempt to read it.
+
+- `enable_bypass_cache_with_threshold` - allows to skip cache completely in case the requested read range exceeds the threshold. Default: `false`. This threshold can be defined by `bypass_cache_threashold`. Default: `268435456` (`256Mi`).
 
 - `do_not_evict_index_and_mark_files` - do not evict small frequently used files according to cache policy. Default: `false`. This setting was added in version 22.8. If you used filesystem cache before this version, then it will not work on versions starting from 22.8 if this setting is set to `true`. If you want to use this setting, clear old cache created before version 22.8 before upgrading.
 
-- `max_file_segment_size` - a maximum size of a single cache file in bytes or in readable format (`ki, Mi, Gi, etc`, example `10Gi`). Default: `104857600` (`100Mi`).
+- `max_file_segment_size` - a maximum size of a single cache file in bytes or in readable format (`ki, Mi, Gi, etc`, example `10Gi`). Default: `8388608` (`8Mi`).
 
-- `max_elements` - a limit for a number of cache files. Default: `1048576`.
+- `max_elements` - a limit for a number of cache files. Default: `10000000`.
 
 File Cache **query/profile settings**:
 

programs/client/Client.cpp

@@ -790,7 +790,7 @@ bool Client::processWithFuzzing(const String & full_query)
 
                 WriteBufferFromOStream cerr_buf(std::cerr, 4096);
                 fuzz_base->dumpTree(cerr_buf);
-                cerr_buf.next();
+                cerr_buf.finalize();
 
                 fmt::print(
                     stderr,
@@ -928,7 +928,7 @@ bool Client::processWithFuzzing(const String & full_query)
         std::cout << std::endl;
         WriteBufferFromOStream ast_buf(std::cout, 4096);
         formatAST(*query, ast_buf, false /*highlight*/);
-        ast_buf.next();
+        ast_buf.finalize();
         if (const auto * insert = query->as<ASTInsertQuery>())
         {
             /// For inserts with data it's really useful to have the data itself available in the logs, as formatAST doesn't print it

programs/format/Format.cpp

@@ -151,6 +151,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
 
             WriteBufferFromFileDescriptor out(STDOUT_FILENO);
             obfuscateQueries(query, out, obfuscated_words_map, used_nouns, hash_func, is_known_identifier);
+            out.finalize();
         }
         else
         {
@@ -175,7 +176,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
                     {
                         WriteBufferFromOStream res_buf(std::cout, 4096);
                         formatAST(*res, res_buf, hilite, oneline);
-                        res_buf.next();
+                        res_buf.finalize();
                         if (multiple)
                             std::cout << "\n;\n";
                         std::cout << std::endl;
@@ -199,7 +200,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
                             res_cout.write(*s_pos++);
                         }
 
-                        res_cout.next();
+                        res_cout.finalize();
                         if (multiple)
                             std::cout << " \\\n;\n";
                         std::cout << std::endl;

src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.cpp

@@ -239,6 +239,7 @@ QueryPipeline ExternalDictionaryLibraryBridgeHelper::loadKeys(const Block & requ
         WriteBufferFromOStream out_buffer(os);
         auto output_format = getContext()->getOutputFormat(ExternalDictionaryLibraryBridgeHelper::DEFAULT_FORMAT, out_buffer, requested_block.cloneEmpty());
         formatBlock(output_format, requested_block);
+        out_buffer.finalize();
     };
     return QueryPipeline(loadBase(uri, out_stream_callback));
 }

src/Client/ClientBase.cpp

@@ -362,7 +362,7 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu
         std::cout << std::endl;
         WriteBufferFromOStream res_buf(std::cout, 4096);
         formatAST(*res, res_buf);
-        res_buf.next();
+        res_buf.finalize();
         std::cout << std::endl << std::endl;
     }
 

src/Client/QueryFuzzer.cpp

@@ -1244,7 +1244,7 @@ void QueryFuzzer::fuzzMain(ASTPtr & ast)
     std::cout << std::endl;
     WriteBufferFromOStream ast_buf(std::cout, 4096);
     formatAST(*ast, ast_buf, false /*highlight*/);
-    ast_buf.next();
+    ast_buf.finalize();
     std::cout << std::endl << std::endl;
 }
 

src/Common/Exception.cpp

@@ -1,5 +1,6 @@
 #include "Exception.h"
 
+#include <algorithm>
 #include <cstring>
 #include <cxxabi.h>
 #include <cstdlib>
@@ -83,19 +84,22 @@ Exception::Exception(const MessageMasked & msg_masked, int code, bool remote_)
     : Poco::Exception(msg_masked.msg, code)
     , remote(remote_)
 {
+    capture_thread_frame_pointers = thread_frame_pointers;
     handle_error_code(msg_masked.msg, code, remote, getStackFramePointers());
 }
 
 Exception::Exception(MessageMasked && msg_masked, int code, bool remote_)
     : Poco::Exception(msg_masked.msg, code)
     , remote(remote_)
 {
+    capture_thread_frame_pointers = thread_frame_pointers;
     handle_error_code(message(), code, remote, getStackFramePointers());
 }
 
 Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc)
     : Poco::Exception(exc.displayText(), ErrorCodes::POCO_EXCEPTION)
 {
+    capture_thread_frame_pointers = thread_frame_pointers;
 #ifdef STD_EXCEPTION_HAS_STACK_TRACE
     auto * stack_trace_frames = exc.get_stack_trace_frames();
     auto stack_trace_size = exc.get_stack_trace_size();
@@ -107,6 +111,7 @@ Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc)
 Exception::Exception(CreateFromSTDTag, const std::exception & exc)
     : Poco::Exception(demangle(typeid(exc).name()) + ": " + String(exc.what()), ErrorCodes::STD_EXCEPTION)
 {
+    capture_thread_frame_pointers = thread_frame_pointers;
 #ifdef STD_EXCEPTION_HAS_STACK_TRACE
     auto * stack_trace_frames = exc.get_stack_trace_frames();
     auto stack_trace_size = exc.get_stack_trace_size();
@@ -153,7 +158,17 @@ std::string Exception::getStackTraceString() const
     auto * stack_trace_frames = get_stack_trace_frames();
     auto stack_trace_size = get_stack_trace_size();
     __msan_unpoison(stack_trace_frames, stack_trace_size * sizeof(stack_trace_frames[0]));
-    return StackTrace::toString(stack_trace_frames, 0, stack_trace_size);
+    String thread_stack_trace;
+    std::for_each(capture_thread_frame_pointers.rbegin(), capture_thread_frame_pointers.rend(),
+        [&thread_stack_trace](StackTrace::FramePointers & frame_pointers)
+        {
+            thread_stack_trace +=
+                "\nJob's origin stack trace:\n" +
+                StackTrace::toString(frame_pointers.data(), 0, std::ranges::find(frame_pointers, nullptr) - frame_pointers.begin());
+        }
+    );
+
+    return StackTrace::toString(stack_trace_frames, 0, stack_trace_size) + thread_stack_trace;
 #else
     return trace.toString();
 #endif
@@ -185,6 +200,9 @@ Exception::FramePointers Exception::getStackFramePointers() const
     return frame_pointers;
 }
 
+thread_local bool Exception::enable_job_stack_trace = false;
+thread_local std::vector<StackTrace::FramePointers> Exception::thread_frame_pointers = {};
+
 
 void throwFromErrno(const std::string & s, int code, int the_errno)
 {

src/Common/Exception.h

@@ -25,18 +25,27 @@ class Exception : public Poco::Exception
 public:
     using FramePointers = std::vector<void *>;
 
-    Exception() = default;
+    Exception()
+    {
+        capture_thread_frame_pointers = thread_frame_pointers;
+    }
 
     Exception(const PreformattedMessage & msg, int code): Exception(msg.text, code)
     {
+        capture_thread_frame_pointers = thread_frame_pointers;
         message_format_string = msg.format_string;
     }
 
     Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.text), code)
     {
+        capture_thread_frame_pointers = thread_frame_pointers;
         message_format_string = msg.format_string;
     }
 
+    /// Collect call stacks of all previous jobs' schedulings leading to this thread job's execution
+    static thread_local bool enable_job_stack_trace;
+    static thread_local std::vector<StackTrace::FramePointers> thread_frame_pointers;
+
 protected:
     // used to remove the sensitive information from exceptions if query_masking_rules is configured
     struct MessageMasked
@@ -66,6 +75,7 @@ class Exception : public Poco::Exception
     Exception(int code, T && message)
         : Exception(message, code)
     {
+        capture_thread_frame_pointers = thread_frame_pointers;
         message_format_string = tryGetStaticFormatString(message);
     }
 
@@ -80,6 +90,7 @@ class Exception : public Poco::Exception
     Exception(int code, FormatStringHelper<Args...> fmt, Args &&... args)
         : Exception(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code)
     {
+        capture_thread_frame_pointers = thread_frame_pointers;
         message_format_string = fmt.message_format_string;
     }
 
@@ -131,6 +142,8 @@ class Exception : public Poco::Exception
 
 protected:
     std::string_view message_format_string;
+    /// Local copy of static per-thread thread_frame_pointers, should be mutable to be unpoisoned on printout
+    mutable std::vector<StackTrace::FramePointers> capture_thread_frame_pointers;
 };
 
 

src/Common/StackTrace.cpp

@@ -412,6 +412,21 @@ void StackTrace::toStringEveryLine(std::function<void(std::string_view)> callbac
     toStringEveryLineImpl(true, {frame_pointers, offset, size}, std::move(callback));
 }
 
+void StackTrace::toStringEveryLine(const FramePointers & frame_pointers, std::function<void(std::string_view)> callback)
+{
+    toStringEveryLineImpl(true, {frame_pointers, 0, static_cast<size_t>(std::ranges::find(frame_pointers, nullptr) - frame_pointers.begin())}, std::move(callback));
+}
+
+void StackTrace::toStringEveryLine(void ** frame_pointers_raw, size_t offset, size_t size, std::function<void(std::string_view)> callback)
+{
+    __msan_unpoison(frame_pointers_raw, size * sizeof(*frame_pointers_raw));
+
+    StackTrace::FramePointers frame_pointers{};
+    std::copy_n(frame_pointers_raw, size, frame_pointers.begin());
+
+    toStringEveryLineImpl(true, {frame_pointers, offset, size}, std::move(callback));
+}
+
 using StackTraceCache = std::map<StackTraceTriple, String, std::less<>>;
 
 static StackTraceCache & cacheInstance()

src/Common/StackTrace.h

@@ -65,6 +65,8 @@ class StackTrace
     static void symbolize(const FramePointers & frame_pointers, size_t offset, size_t size, StackTrace::Frames & frames);
 
     void toStringEveryLine(std::function<void(std::string_view)> callback) const;
+    static void toStringEveryLine(const FramePointers & frame_pointers, std::function<void(std::string_view)> callback);
+    static void toStringEveryLine(void ** frame_pointers_raw, size_t offset, size_t size, std::function<void(std::string_view)> callback);
 
     /// Displaying the addresses can be disabled for security reasons.
     /// If you turn off addresses, it will be more secure, but we will be unable to help you with debugging.