Merge branch 'master' into fix-ip-aggregate-state

README.md

@@ -22,12 +22,14 @@ curl https://clickhouse.com/ | sh
 
 ## Upcoming Events
 
-* [**v23.5 Release Webinar**](https://clickhouse.com/company/events/v23-5-release-webinar?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-05) - Jun 8 - 23.5 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
-* [**ClickHouse Meetup in Bangalore**](https://www.meetup.com/clickhouse-bangalore-user-group/events/293740066/) - Jun 7
-* [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/293426725/) - Jun 7 
-
-
-Also, keep an eye out for upcoming meetups in Amsterdam, Boston, NYC, Beijing, and Toronto. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
+* [**v23.6 Release Webinar**](https://clickhouse.com/company/events/v23-6-release-call?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-06) - Jun 29 - 23.6 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**ClickHouse Meetup in Paris**](https://www.meetup.com/clickhouse-france-user-group/events/294283460) - Jul 4
+* [**ClickHouse Meetup in Boston**](https://www.meetup.com/clickhouse-boston-user-group/events/293913596) - Jul 18
+* [**ClickHouse Meetup in NYC**](https://www.meetup.com/clickhouse-new-york-user-group/events/293913441) - Jul 19
+* [**ClickHouse Meetup in Toronto**](https://www.meetup.com/clickhouse-toronto-user-group/events/294183127) - Jul 20
+* [**ClickHouse Meetup in Singapore**](https://www.meetup.com/clickhouse-singapore-meetup-group/events/294428050/) - Jul 27
+
+Also, keep an eye out for upcoming meetups around the world. Somewhere else you want us to be? Please feel free to reach out to tyler <at> clickhouse <dot> com.
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"

base/base/bit_cast.h

@@ -7,7 +7,13 @@
 
 /** Returns value `from` converted to type `To` while retaining bit representation.
   * `To` and `From` must satisfy `CopyConstructible`.
+  *
   * In contrast to std::bit_cast can cast types of different width.
+  *
+  * Note: for signed types of narrower size, the casted result is zero-extended
+  * instead of sign-extended as with regular static_cast.
+  * For example, -1 Int8 (represented as 0xFF) bit_casted to UInt64
+  * gives 255 (represented as 0x00000000000000FF) instead of 0xFFFFFFFFFFFFFFFF
   */
 template <typename To, typename From>
 std::decay_t<To> bit_cast(const From & from)

base/base/find_symbols.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <string>
+#include <array>
 
 #if defined(__SSE2__)
     #include <emmintrin.h>

base/base/interpolate.h

@@ -11,3 +11,8 @@ constexpr double interpolateExponential(double min, double max, double ratio)
     assert(min > 0 && ratio >= 0 && ratio <= 1);
     return min * std::pow(max / min, ratio);
 }
+
+constexpr double interpolateLinear(double min, double max, double ratio)
+{
+    return std::lerp(min, max, ratio);
+}

base/poco/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h

@@ -85,7 +85,7 @@ namespace Net
     ///                </options>
     ///            </privateKeyPassphraseHandler>
     ///            <invalidCertificateHandler>
-    ///                 <name>ConsoleCertificateHandler</name>
+    ///                 <name>RejectCertificateHandler</name>
     ///            </invalidCertificateHandler>
     ///            <cacheSessions>true|false</cacheSessions>
     ///            <sessionIdContext>someString</sessionIdContext> <!-- server only -->
@@ -186,7 +186,7 @@ namespace Net
         ///
         /// Valid initialization code would be:
         ///     SharedPtr<PrivateKeyPassphraseHandler> pConsoleHandler = new KeyConsoleHandler;
-        ///     SharedPtr<InvalidCertificateHandler> pInvalidCertHandler = new ConsoleCertificateHandler;
+        ///     SharedPtr<InvalidCertificateHandler> pInvalidCertHandler = new RejectCertificateHandler;
         ///     Context::Ptr pContext = new Context(Context::SERVER_USE, "any.pem", "any.pem", "rootcert.pem", Context::VERIFY_RELAXED, 9, false, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
         ///     SSLManager::instance().initializeServer(pConsoleHandler, pInvalidCertHandler, pContext);
 
@@ -203,7 +203,7 @@ namespace Net
         ///
         /// Valid initialization code would be:
         ///     SharedPtr<PrivateKeyPassphraseHandler> pConsoleHandler = new KeyConsoleHandler;
-        ///     SharedPtr<InvalidCertificateHandler> pInvalidCertHandler = new ConsoleCertificateHandler;
+        ///     SharedPtr<InvalidCertificateHandler> pInvalidCertHandler = new RejectCertificateHandler;
         ///     Context::Ptr pContext = new Context(Context::CLIENT_USE, "", "", "rootcert.pem", Context::VERIFY_RELAXED, 9, false, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
         ///     SSLManager::instance().initializeClient(pConsoleHandler, pInvalidCertHandler, pContext);
 

base/poco/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp

@@ -13,7 +13,6 @@
 
 
 #include "Poco/Net/CertificateHandlerFactoryMgr.h"
-#include "Poco/Net/ConsoleCertificateHandler.h"
 #include "Poco/Net/AcceptCertificateHandler.h"
 #include "Poco/Net/RejectCertificateHandler.h"
 
@@ -24,7 +23,6 @@ namespace Net {
 
 CertificateHandlerFactoryMgr::CertificateHandlerFactoryMgr()
 {
-	setFactory("ConsoleCertificateHandler", new CertificateHandlerFactoryImpl<ConsoleCertificateHandler>());
 	setFactory("AcceptCertificateHandler", new CertificateHandlerFactoryImpl<AcceptCertificateHandler>());
 	setFactory("RejectCertificateHandler", new CertificateHandlerFactoryImpl<RejectCertificateHandler>());
 }

base/poco/NetSSL_OpenSSL/src/SSLManager.cpp

@@ -46,7 +46,7 @@ const std::string SSLManager::CFG_PREFER_SERVER_CIPHERS("preferServerCiphers");
 const std::string SSLManager::CFG_DELEGATE_HANDLER("privateKeyPassphraseHandler.name");
 const std::string SSLManager::VAL_DELEGATE_HANDLER("KeyConsoleHandler");
 const std::string SSLManager::CFG_CERTIFICATE_HANDLER("invalidCertificateHandler.name");
-const std::string SSLManager::VAL_CERTIFICATE_HANDLER("ConsoleCertificateHandler");
+const std::string SSLManager::VAL_CERTIFICATE_HANDLER("RejectCertificateHandler");
 const std::string SSLManager::CFG_SERVER_PREFIX("openSSL.server.");
 const std::string SSLManager::CFG_CLIENT_PREFIX("openSSL.client.");
 const std::string SSLManager::CFG_CACHE_SESSIONS("cacheSessions");

contrib/CMakeLists.txt

@@ -146,7 +146,7 @@ add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
 add_contrib (cassandra-cmake cassandra) # requires: libuv
 if (NOT OS_DARWIN)
     add_contrib (curl-cmake curl)
-    add_contrib (azure-cmake azure)
+    add_contrib (azure-cmake azure) # requires: curl
     add_contrib (sentry-native-cmake sentry-native) # requires: curl
 endif()
 add_contrib (fmtlib-cmake fmtlib)
@@ -157,7 +157,7 @@ add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
 add_contrib (nats-io-cmake nats-io)
 add_contrib (isa-l-cmake isa-l)
 add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l
-add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3
+add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arrow, libhdfs3
 add_contrib (cppkafka-cmake cppkafka)
 add_contrib (libpqxx-cmake libpqxx)
 add_contrib (libpq-cmake libpq)

contrib/arrow-cmake/CMakeLists.txt

@@ -31,12 +31,12 @@ endif()
 
 set (CMAKE_CXX_STANDARD 17)
 
-set(ARROW_VERSION "6.0.1")
+set(ARROW_VERSION "11.0.0")
 string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
 
-set(ARROW_VERSION_MAJOR "6")
+set(ARROW_VERSION_MAJOR "11")
 set(ARROW_VERSION_MINOR "0")
-set(ARROW_VERSION_PATCH "1")
+set(ARROW_VERSION_PATCH "0")
 
 if(ARROW_VERSION_MAJOR STREQUAL "0")
     # Arrow 0.x.y => SO version is "x", full SO version is "x.y.0"
@@ -116,43 +116,79 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A
 # ARROW_ORC + adapters/orc/CMakefiles
 set(ORC_SRCS
         "${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h"
-        "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc"
-        "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc"
-        "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc"
-        "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc"
-        "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc"
-        "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc"
-        "${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
-        "${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
-        "${ORC_SOURCE_SRC_DIR}/Reader.cc"
+        "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
+        "${ORC_SOURCE_SRC_DIR}/Adaptor.cc"
+        "${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in"
+        "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc"
+        "${ORC_SOURCE_SRC_DIR}/BlockBuffer.hh"
+        "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc"
+        "${ORC_SOURCE_SRC_DIR}/BloomFilter.hh"
+        "${ORC_SOURCE_SRC_DIR}/Bpacking.hh"
+        "${ORC_SOURCE_SRC_DIR}/BpackingDefault.cc"
+        "${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh"
         "${ORC_SOURCE_SRC_DIR}/ByteRLE.cc"
+        "${ORC_SOURCE_SRC_DIR}/ByteRLE.hh"
+        "${ORC_SOURCE_SRC_DIR}/CMakeLists.txt"
         "${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc"
         "${ORC_SOURCE_SRC_DIR}/ColumnReader.cc"
+        "${ORC_SOURCE_SRC_DIR}/ColumnReader.hh"
         "${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc"
+        "${ORC_SOURCE_SRC_DIR}/ColumnWriter.hh"
         "${ORC_SOURCE_SRC_DIR}/Common.cc"
         "${ORC_SOURCE_SRC_DIR}/Compression.cc"
+        "${ORC_SOURCE_SRC_DIR}/Compression.hh"
+        "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.cc"
+        "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.hh"
+        "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.cc"
+        "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.hh"
+        "${ORC_SOURCE_SRC_DIR}/Dispatch.hh"
+        "${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
         "${ORC_SOURCE_SRC_DIR}/Int128.cc"
         "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc"
+        "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.hh"
         "${ORC_SOURCE_SRC_DIR}/MemoryPool.cc"
+        "${ORC_SOURCE_SRC_DIR}/Murmur3.cc"
+        "${ORC_SOURCE_SRC_DIR}/Murmur3.hh"
+        "${ORC_SOURCE_SRC_DIR}/Options.hh"
+        "${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
         "${ORC_SOURCE_SRC_DIR}/RLE.cc"
+        "${ORC_SOURCE_SRC_DIR}/RLE.hh"
+        "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc"
+        "${ORC_SOURCE_SRC_DIR}/RLEV2Util.hh"
         "${ORC_SOURCE_SRC_DIR}/RLEv1.cc"
+        "${ORC_SOURCE_SRC_DIR}/RLEv1.hh"
+        "${ORC_SOURCE_SRC_DIR}/RLEv2.hh"
+        "${ORC_SOURCE_SRC_DIR}/Reader.cc"
+        "${ORC_SOURCE_SRC_DIR}/Reader.hh"
         "${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc"
         "${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc"
-        "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc"
+        "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.cc"
+        "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.hh"
         "${ORC_SOURCE_SRC_DIR}/Statistics.cc"
+        "${ORC_SOURCE_SRC_DIR}/Statistics.hh"
         "${ORC_SOURCE_SRC_DIR}/StripeStream.cc"
+        "${ORC_SOURCE_SRC_DIR}/StripeStream.hh"
         "${ORC_SOURCE_SRC_DIR}/Timezone.cc"
+        "${ORC_SOURCE_SRC_DIR}/Timezone.hh"
         "${ORC_SOURCE_SRC_DIR}/TypeImpl.cc"
+        "${ORC_SOURCE_SRC_DIR}/TypeImpl.hh"
+        "${ORC_SOURCE_SRC_DIR}/Utils.hh"
         "${ORC_SOURCE_SRC_DIR}/Vector.cc"
         "${ORC_SOURCE_SRC_DIR}/Writer.cc"
-        "${ORC_SOURCE_SRC_DIR}/Adaptor.cc"
-        "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc"
-        "${ORC_SOURCE_SRC_DIR}/Murmur3.cc"
-        "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc"
-        "${ORC_SOURCE_SRC_DIR}/wrap/orc-proto-wrapper.cc"
         "${ORC_SOURCE_SRC_DIR}/io/InputStream.cc"
+        "${ORC_SOURCE_SRC_DIR}/io/InputStream.hh"
         "${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc"
-        "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
+        "${ORC_SOURCE_SRC_DIR}/io/OutputStream.hh"
+        "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc"
+        "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.hh"
+        "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc"
+        "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc"
+        "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.hh"
+        "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc"
+        "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.hh"
+        "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc"
+        "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.hh"
+        "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc"
         )
 
 add_library(_orc ${ORC_SRCS})
@@ -478,6 +514,10 @@ if (SANITIZE STREQUAL "undefined")
     target_compile_options(_arrow PRIVATE -fno-sanitize=undefined)
 endif ()
 
+# Define Thrift version for parquet (we use 0.16.0)
+add_definitions(-DPARQUET_THRIFT_VERSION_MAJOR=0)
+add_definitions(-DPARQUET_THRIFT_VERSION_MINOR=16)
+
 # === tools
 
 set(TOOLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/tools/parquet")

contrib/azure-cmake/CMakeLists.txt

@@ -1,6 +1,6 @@
 option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES})
 
-if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD OR (NOT ARCH_AMD64))
+if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD)
     message(STATUS "Not using Azure blob storage")
     return()
 endif()

contrib/libhdfs3-cmake/CMakeLists.txt

@@ -1,11 +1,11 @@
-if(NOT ARCH_AARCH64 AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
+if(NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE AND NOT ARCH_S390X)
     option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
 elseif(ENABLE_HDFS)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")
 endif()
 
 if(NOT ENABLE_HDFS)
-    message(STATUS "Not using hdfs")
+    message(STATUS "Not using HDFS")
     return()
 endif()