Merge branch 'master' into fix-OR

CHANGELOG.md

@@ -90,7 +90,6 @@
 * Allow reading some unusual configuration of chunks from Arrow/Parquet formats. [#54370](https://github.com/ClickHouse/ClickHouse/pull/54370) ([Arthur Passos](https://github.com/arthurpassos)).
 * Add `STD` alias to `stddevPop` function for MySQL compatibility. Closes [#54274](https://github.com/ClickHouse/ClickHouse/issues/54274). [#54382](https://github.com/ClickHouse/ClickHouse/pull/54382) ([Nikolay Degterinsky](https://github.com/evillique)).
 * Add `addDate` function for compatibility with MySQL and `subDate` for consistency. Reference [#54275](https://github.com/ClickHouse/ClickHouse/issues/54275). [#54400](https://github.com/ClickHouse/ClickHouse/pull/54400) ([Nikolay Degterinsky](https://github.com/evillique)).
-* Support `SAMPLE BY` for views. [#54477](https://github.com/ClickHouse/ClickHouse/pull/54477) ([Azat Khuzhin](https://github.com/azat)).
 * Add `modification_time` into `system.detached_parts`. [#54506](https://github.com/ClickHouse/ClickHouse/pull/54506) ([Azat Khuzhin](https://github.com/azat)).
 * Added a setting `splitby_max_substrings_includes_remaining_string` which controls if functions "splitBy*()" with argument "max_substring" > 0 include the remaining string (if any) in the result array (Python/Spark semantics) or not. The default behavior does not change. [#54518](https://github.com/ClickHouse/ClickHouse/pull/54518) ([Robert Schulze](https://github.com/rschu1ze)).
 * Better integer types inference for `Int64`/`UInt64` fields. Continuation of [#53003](https://github.com/ClickHouse/ClickHouse/pull/53003). Now it works also for nested types like Arrays of Arrays and for functions like `map/tuple`. Issue: [#51236](https://github.com/ClickHouse/ClickHouse/issues/51236). [#54553](https://github.com/ClickHouse/ClickHouse/pull/54553) ([Kruglov Pavel](https://github.com/Avogar)).
@@ -442,7 +441,7 @@
 * Add new setting `disable_url_encoding` that allows to disable decoding/encoding path in uri in URL engine. [#52337](https://github.com/ClickHouse/ClickHouse/pull/52337) ([Kruglov Pavel](https://github.com/Avogar)).
 
 #### Performance Improvement
-* Enable automatic selection of the sparse serialization format by default. It improves performance. The format is supported since version 22.1. After this change, downgrading to versions older than 22.1 might not be possible. You can turn off the usage of the sparse serialization format by providing the `ratio_of_defaults_for_sparse_serialization = 1` setting for your MergeTree tables. [#49631](https://github.com/ClickHouse/ClickHouse/pull/49631) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Enable automatic selection of the sparse serialization format by default. It improves performance. The format is supported since version 22.1. After this change, downgrading to versions older than 22.1 might not be possible. A downgrade may require to set `ratio_of_defaults_for_sparse_serialization=0.9375` [55153](https://github.com/ClickHouse/ClickHouse/issues/55153). You can turn off the usage of the sparse serialization format by providing the `ratio_of_defaults_for_sparse_serialization = 1` setting for your MergeTree tables. [#49631](https://github.com/ClickHouse/ClickHouse/pull/49631) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Enable `move_all_conditions_to_prewhere` and `enable_multiple_prewhere_read_steps` settings by default. [#46365](https://github.com/ClickHouse/ClickHouse/pull/46365) ([Alexander Gololobov](https://github.com/davenger)).
 * Improves performance of some queries by tuning allocator. [#46416](https://github.com/ClickHouse/ClickHouse/pull/46416) ([Azat Khuzhin](https://github.com/azat)).
 * Now we use fixed-size tasks in `MergeTreePrefetchedReadPool` as in `MergeTreeReadPool`. Also from now we use connection pool for S3 requests. [#49732](https://github.com/ClickHouse/ClickHouse/pull/49732) ([Nikita Taranov](https://github.com/nickitat)).

README.md

@@ -26,7 +26,6 @@ curl https://clickhouse.com/ | sh
 * [**ClickHouse Meetup in Amsterdam**](https://www.meetup.com/clickhouse-netherlands-user-group/events/296334590/) - Oct 31
 * [**ClickHouse Meetup in Beijing**](https://www.meetup.com/clickhouse-beijing-user-group/events/296334856/) - Nov 4
 * [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/296334923/) - Nov 8
-* [**ClickHouse Meetup in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/296487639/) - Nov 8
 * [**ClickHouse Meetup in Singapore**](https://www.meetup.com/clickhouse-singapore-meetup-group/events/296334976/) - Nov 15
 * [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/296488501/) - Nov 30
 * [**ClickHouse Meetup in NYC**](https://www.meetup.com/clickhouse-new-york-user-group/events/296488779/) - Dec 11

base/poco/MongoDB/include/Poco/MongoDB/Database.h

@@ -70,7 +70,7 @@ namespace MongoDB
         Document::Ptr queryBuildInfo(Connection & connection) const;
         /// Queries server build info (all wire protocols)
 
-        Document::Ptr queryServerHello(Connection & connection) const;
+        Document::Ptr queryServerHello(Connection & connection, bool old = false) const;
         /// Queries hello response from server (all wire protocols)
 
         Int64 count(Connection & connection, const std::string & collectionName) const;

base/poco/MongoDB/src/Database.cpp

@@ -356,11 +356,19 @@ Document::Ptr Database::queryBuildInfo(Connection& connection) const
 }
 
 
-Document::Ptr Database::queryServerHello(Connection& connection) const
+Document::Ptr Database::queryServerHello(Connection& connection, bool old) const
 {
 	// hello can be issued on "config" system database
 	Poco::SharedPtr<Poco::MongoDB::QueryRequest> request = createCommand();
-	request->selector().add("hello", 1);
+
+	// 'hello' command was previously called 'isMaster'
+	std::string command_name;
+	if (old)
+		command_name = "isMaster";
+	else
+		command_name = "hello";
+
+	request->selector().add(command_name, 1);
 
 	Poco::MongoDB::ResponseMessage response;
 	connection.sendRequest(*request, response);

contrib/curl-cmake/CMakeLists.txt

@@ -64,6 +64,7 @@ set (SRCS
     "${LIBRARY_DIR}/lib/hostsyn.c"
     "${LIBRARY_DIR}/lib/hsts.c"
     "${LIBRARY_DIR}/lib/http.c"
+    "${LIBRARY_DIR}/lib/http1.c"
     "${LIBRARY_DIR}/lib/http2.c"
     "${LIBRARY_DIR}/lib/http_aws_sigv4.c"
     "${LIBRARY_DIR}/lib/http_chunks.c"

contrib/googletest-cmake/CMakeLists.txt

@@ -6,20 +6,14 @@ target_compile_definitions (_gtest PUBLIC GTEST_HAS_POSIX_RE=0)
 target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/googletest/include")
 target_include_directories(_gtest PRIVATE "${SRC_DIR}/googletest")
 
-add_library(_gtest_main "${SRC_DIR}/googletest/src/gtest_main.cc")
-set_target_properties(_gtest_main PROPERTIES VERSION "1.0.0")
-target_link_libraries(_gtest_main PUBLIC _gtest)
-
-add_library(_gtest_all INTERFACE)
-target_link_libraries(_gtest_all INTERFACE _gtest _gtest_main)
-add_library(ch_contrib::gtest_all ALIAS _gtest_all)
-
+add_library(ch_contrib::gtest ALIAS _gtest)
 
 add_library(_gmock "${SRC_DIR}/googlemock/src/gmock-all.cc")
 set_target_properties(_gmock PROPERTIES VERSION "1.0.0")
 target_compile_definitions (_gmock PUBLIC GTEST_HAS_POSIX_RE=0)
 target_include_directories(_gmock SYSTEM PUBLIC "${SRC_DIR}/googlemock/include" "${SRC_DIR}/googletest/include")
 target_include_directories(_gmock PRIVATE "${SRC_DIR}/googlemock")
+target_link_libraries(_gmock PUBLIC _gtest)
 
 add_library(_gmock_main "${SRC_DIR}/googlemock/src/gmock_main.cc")
 set_target_properties(_gmock_main PROPERTIES VERSION "1.0.0")

docker/test/base/setup_export_logs.sh

@@ -15,8 +15,8 @@ CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
 # Pre-configured destination cluster, where to export the data
 CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
 
-EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, "}
-EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
+EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name String, instance_type String, instance_id String, "}
+EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS Int32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
 EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name, "}
 
 function __set_connection_args
@@ -125,9 +125,9 @@ function setup_logs_replication
     echo 'Create %_log tables'
     clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
     do
-        # Calculate hash of its structure. Note: 1 is the version of extra columns - increment it if extra columns are changed:
+        # Calculate hash of its structure. Note: 4 is the version of extra columns - increment it if extra columns are changed:
         hash=$(clickhouse-client --query "
-            SELECT sipHash64(1, groupArray((name, type)))
+            SELECT sipHash64(4, groupArray((name, type)))
             FROM (SELECT name, type FROM system.columns
                 WHERE database = 'system' AND table = '$table'
                 ORDER BY position)

docker/test/integration/mysql_java_client/Dockerfile

@@ -10,6 +10,5 @@ RUN curl -L -o /mysql-connector-j-${ver}.jar https://repo1.maven.org/maven2/com/
 ENV CLASSPATH=$CLASSPATH:/mysql-connector-j-${ver}.jar
 
 WORKDIR /jdbc
-COPY Test.java Test.java
-COPY PreparedStatementsTest.java PreparedStatementsTest.java
-RUN javac Test.java PreparedStatementsTest.java
+COPY MySQLJavaClientTest.java MySQLJavaClientTest.java
+RUN javac MySQLJavaClientTest.java

docker/test/integration/mysql_java_client/PreparedStatementsTest.java → docker/test/integration/mysql_java_client/MySQLJavaClientTest.java

@@ -2,14 +2,15 @@
 
 import java.sql.*;
 
-public class PreparedStatementsTest {
+public class MySQLJavaClientTest {
     public static void main(String[] args) {
         int i = 0;
         String host = "127.0.0.1";
         String port = "9004";
         String user = "default";
         String password = "";
         String database = "default";
+        String binary = "false";
         while (i < args.length) {
             switch (args[i]) {
                 case "--host":
@@ -27,16 +28,19 @@ public static void main(String[] args) {
                 case "--database":
                     database = args[++i];
                     break;
+                case "--binary":
+                    binary = args[++i];
+                    break;
                 default:
                     i++;
                     break;
             }
         }
 
-        // useServerPrepStmts uses COM_STMT_PREPARE and COM_STMT_EXECUTE
-        // instead of COM_QUERY which allows us to test the binary protocol
-        String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=true",
-                host, port, database);
+        // useServerPrepStmts=true -> COM_STMT_PREPARE + COM_STMT_EXECUTE -> binary
+        // useServerPrepStmts=false -> COM_QUERY -> text
+        String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s?useSSL=false&useServerPrepStmts=%s",
+                host, port, database, binary);
 
         try {
             Class.forName("com.mysql.cj.jdbc.Driver");
@@ -49,6 +53,7 @@ public static void main(String[] args) {
             testDateTypes(conn);
             testUnusualDateTime64Scales(conn);
             testDateTimeTimezones(conn);
+            testSuspiciousNullableLowCardinalityTypes(conn);
             conn.close();
         } catch (Exception e) {
             e.printStackTrace();
@@ -58,7 +63,7 @@ public static void main(String[] args) {
 
     private static void testSimpleDataTypes(Connection conn) throws SQLException {
         System.out.println("### testSimpleDataTypes");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_simple_data_types").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM simple_data_types").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -83,7 +88,7 @@ private static void testSimpleDataTypes(Connection conn) throws SQLException {
 
     private static void testStringTypes(Connection conn) throws SQLException {
         System.out.println("### testStringTypes");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_string_types").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM string_types").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -97,7 +102,7 @@ private static void testStringTypes(Connection conn) throws SQLException {
 
     private static void testLowCardinalityAndNullableTypes(Connection conn) throws SQLException {
         System.out.println("### testLowCardinalityAndNullableTypes");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_low_cardinality_and_nullable_types").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM low_cardinality_and_nullable_types").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -111,7 +116,7 @@ private static void testLowCardinalityAndNullableTypes(Connection conn) throws S
 
     private static void testDecimalTypes(Connection conn) throws SQLException {
         System.out.println("### testDecimalTypes");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_decimal_types").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM decimal_types").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -127,7 +132,7 @@ private static void testDecimalTypes(Connection conn) throws SQLException {
 
     private static void testDateTypes(Connection conn) throws SQLException {
         System.out.println("### testDateTypes");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_date_types").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM date_types").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -143,7 +148,7 @@ private static void testDateTypes(Connection conn) throws SQLException {
 
     private static void testUnusualDateTime64Scales(Connection conn) throws SQLException {
         System.out.println("### testUnusualDateTime64Scales");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_unusual_datetime64_scales").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM unusual_datetime64_scales").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -160,7 +165,7 @@ private static void testUnusualDateTime64Scales(Connection conn) throws SQLExcep
 
     private static void testDateTimeTimezones(Connection conn) throws SQLException {
         System.out.println("### testDateTimeTimezones");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_datetime_timezones").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM datetime_timezones").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -172,7 +177,7 @@ private static void testDateTimeTimezones(Connection conn) throws SQLException {
 
     private static void testMiscTypes(Connection conn) throws SQLException {
         System.out.println("### testMiscTypes");
-        ResultSet rs = conn.prepareStatement("SELECT * FROM ps_misc_types").executeQuery();
+        ResultSet rs = conn.prepareStatement("SELECT * FROM misc_types").executeQuery();
         int rowNum = 1;
         while (rs.next()) {
             System.out.printf("Row #%d\n", rowNum++);
@@ -184,6 +189,20 @@ private static void testMiscTypes(Connection conn) throws SQLException {
         System.out.println();
     }
 
+    private static void testSuspiciousNullableLowCardinalityTypes(Connection conn) throws SQLException {
+        System.out.println("### testSuspiciousNullableLowCardinalityTypes");
+        String query = "SELECT * FROM suspicious_nullable_low_cardinality_types";
+        ResultSet rs = conn.prepareStatement(query).executeQuery();
+        int rowNum = 1;
+        while (rs.next()) {
+            System.out.printf("Row #%d\n", rowNum++);
+            System.out.printf("%s, value: %s\n", getMysqlType(rs, "f"), rs.getFloat("f"));
+            System.out.printf("%s, value: %s\n", getMysqlType(rs, "d"), rs.getDate("d"));
+            System.out.printf("%s, value: %s\n", getMysqlType(rs, "dt"), rs.getTimestamp("dt"));
+        }
+        System.out.println();
+    }
+
     private static String getMysqlType(ResultSet rs, String columnLabel) throws SQLException {
         ResultSetMetaData meta = rs.getMetaData();
         return String.format("%s type is %s", columnLabel,

docker/test/integration/runner/compose/docker_compose_minio.yml

@@ -2,9 +2,7 @@ version: '2.3'
 
 services:
   minio1:
-    # Newer version of minio results in such errors:
-    # "AWSErrorMarshaller: Encountered AWSError 'InternalError': We encountered an internal error, please try again"
-    image: minio/minio:RELEASE.2021-09-23T04-46-24Z
+    image: minio/minio:RELEASE.2023-09-30T07-02-29Z
     volumes:
       - data1-1:/data1
       - ${MINIO_CERTS_DIR:-}:/certs