Merge branch '23.3' into backport/23.3/55309

cmake/autogenerated_versions.txt

@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54472)
 SET(VERSION_MAJOR 23)
 SET(VERSION_MINOR 3)
-SET(VERSION_PATCH 14)
-SET(VERSION_GITHASH 25635e2755138265c6ac1d99fbedc62298ce629d)
-SET(VERSION_DESCRIBE v23.3.14.1-lts)
-SET(VERSION_STRING 23.3.14.1)
+SET(VERSION_PATCH 15)
+SET(VERSION_GITHASH c8f4ba52c65d2a2ee4ec58c5e44accfb26b6e535)
+SET(VERSION_DESCRIBE v23.3.15.1-lts)
+SET(VERSION_STRING 23.3.15.1)
 # end of autochange

docker/packager/binary/Dockerfile

@@ -50,6 +50,8 @@ ENV PATH="/rust/cargo/env:${PATH}"
 ENV PATH="/rust/cargo/bin:${PATH}"
 RUN curl https://sh.rustup.rs -sSf | bash -s -- -y && \
     chmod 777 -R /rust && \
+    rustup toolchain install 1.71.0 && \
+    rustup default 1.71.0 && \
     rustup target add aarch64-unknown-linux-gnu && \
     rustup target add x86_64-apple-darwin && \
     rustup target add x86_64-unknown-freebsd && \

docs/en/sql-reference/statements/alter/partition.md

@@ -109,6 +109,7 @@ For the query to run successfully, the following conditions must be met:
 
 -   Both tables must have the same structure.
 -   Both tables must have the same partition key, the same order by key and the same primary key.
+-   Both tables must have the same indices and projections.
 -   Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
 
 ## REPLACE PARTITION
@@ -123,6 +124,7 @@ For the query to run successfully, the following conditions must be met:
 
 -   Both tables must have the same structure.
 -   Both tables must have the same partition key, the same order by key and the same primary key.
+-   Both tables must have the same indices and projections.
 -   Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
 
 ## MOVE PARTITION TO TABLE
@@ -137,6 +139,7 @@ For the query to run successfully, the following conditions must be met:
 
 -   Both tables must have the same structure.
 -   Both tables must have the same partition key, the same order by key and the same primary key.
+-   Both tables must have the same indices and projections.
 -   Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables).
 -   Both tables must be the same engine family (replicated or non-replicated).
 

src/Interpreters/QueryNormalizer.cpp

@@ -68,6 +68,10 @@ class RestoreAliasOnExitScope
 
 void QueryNormalizer::visit(ASTIdentifier & node, ASTPtr & ast, Data & data)
 {
+    /// We do handle cycles via tracking current_asts
+    /// but in case of bug in that tricky logic we need to prevent stack overflow
+    checkStackSize();
+
     auto & current_asts = data.current_asts;
     String & current_alias = data.current_alias;
 

src/Storages/MergeTree/MergeTreeData.cpp

@@ -7019,6 +7019,28 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour
     if (query_to_string(my_snapshot->getPrimaryKeyAST()) != query_to_string(src_snapshot->getPrimaryKeyAST()))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different primary key");
 
+    const auto check_definitions = [](const auto & my_descriptions, const auto & src_descriptions)
+    {
+        if (my_descriptions.size() != src_descriptions.size())
+            return false;
+
+        std::unordered_set<std::string> my_query_strings;
+        for (const auto & description : my_descriptions)
+            my_query_strings.insert(queryToString(description.definition_ast));
+
+        for (const auto & src_description : src_descriptions)
+            if (!my_query_strings.contains(queryToString(src_description.definition_ast)))
+                return false;
+
+        return true;
+    };
+
+    if (!check_definitions(my_snapshot->getSecondaryIndices(), src_snapshot->getSecondaryIndices()))
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different secondary indices");
+
+    if (!check_definitions(my_snapshot->getProjections(), src_snapshot->getProjections()))
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Tables have different projections");
+
     return *src_data;
 }
 

src/Storages/MergeTree/MergeTreePartsMover.cpp

@@ -224,10 +224,16 @@ MergeTreePartsMover::TemporaryClonedPart MergeTreePartsMover::clonePart(const Me
         String relative_path = part->getDataPartStorage().getPartDirectory();
         if (disk->exists(path_to_clone + relative_path))
         {
-            throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS,
-                "Cannot clone part {} from '{}' to '{}': path '{}' already exists",
-                part->name, part->getDataPartStorage().getDiskName(), disk->getName(),
+            // If setting is on, we should've already cleaned moving/ dir on startup
+            if (data->allowRemoveStaleMovingParts())
+                throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS,
+                    "Cannot clone part {} from '{}' to '{}': path '{}' already exists",
+                    part->name, part->getDataPartStorage().getDiskName(), disk->getName(),
+                    fullPath(disk, path_to_clone + relative_path));
+
+            LOG_DEBUG(log, "Path {} already exists. Will remove it and clone again",
                 fullPath(disk, path_to_clone + relative_path));
+            disk->removeRecursive(fs::path(path_to_clone) / relative_path / "");
         }
 
         disk->createDirectories(path_to_clone);

tests/queries/0_stateless/02888_attach_partition_from_different_tables.reference

@@ -0,0 +1,11 @@
+1	1
+0	0
+1	1
+2	2
+3	3
+4	4
+5	5
+6	6
+7	7
+8	8
+9	9

tests/queries/0_stateless/02888_attach_partition_from_different_tables.sql

@@ -0,0 +1,90 @@
+-- test different index type
+CREATE TABLE attach_partition_t1 (
+	a UInt32,
+	b String,
+	INDEX bf b TYPE tokenbf_v1(8192, 3, 0) GRANULARITY 1
+)
+ENGINE = MergeTree
+ORDER BY a;
+
+INSERT INTO attach_partition_t1 SELECT number, toString(number) FROM numbers(10);
+
+CREATE TABLE attach_partition_t2 (
+	a UInt32,
+	b String,
+	INDEX bf b TYPE bloom_filter GRANULARITY 1
+)
+ENGINE = MergeTree
+ORDER BY a;
+
+ALTER TABLE attach_partition_t2 ATTACH PARTITION tuple() FROM attach_partition_t1; -- { serverError 36 }
+
+-- test different projection name
+CREATE TABLE attach_partition_t3 (
+	a UInt32,
+	b String,
+  PROJECTION proj
+   (
+       SELECT
+           b,
+           sum(a)
+       GROUP BY b
+   )
+)
+ENGINE = MergeTree
+ORDER BY a;
+
+INSERT INTO attach_partition_t3 SELECT number, toString(number) FROM numbers(10);
+
+CREATE TABLE attach_partition_t4 (
+	a UInt32,
+	b String,
+  PROJECTION differently_named_proj
+   (
+       SELECT
+           b,
+           sum(a)
+       GROUP BY b
+   )
+)
+ENGINE = MergeTree
+ORDER BY a;
+
+ALTER TABLE attach_partition_t4 ATTACH PARTITION tuple() FROM attach_partition_t3; -- { serverError 36 }
+
+-- check attach with same index and projection
+CREATE TABLE attach_partition_t5 (
+	a UInt32,
+	b String,
+  PROJECTION proj
+   (
+       SELECT
+           b,
+           sum(a)
+       GROUP BY b
+   )
+)
+ENGINE = MergeTree
+ORDER BY a;
+
+INSERT INTO attach_partition_t5 SELECT number, toString(number) FROM numbers(10);
+
+
+CREATE TABLE attach_partition_t6 (
+	a UInt32,
+	b String,
+  PROJECTION proj
+   (
+       SELECT
+           b,
+           sum(a)
+       GROUP BY b
+   )
+)
+ENGINE = MergeTree
+ORDER BY a;
+
+ALTER TABLE attach_partition_t6 ATTACH PARTITION tuple() FROM attach_partition_t5;
+
+SELECT * FROM attach_partition_t6 WHERE b = '1';
+SELECT b, sum(a) FROM attach_partition_t6 GROUP BY b ORDER BY b;

tests/queries/0_stateless/02896_cyclic_aliases_crash.reference

@@ -0,0 +1 @@
+1	2	3

tests/queries/0_stateless/02896_cyclic_aliases_crash.sql

@@ -0,0 +1,17 @@
+
+SET max_ast_depth = 10_000_000;
+
+SELECT
+    val,
+    val + 1 as prev,
+    val + prev as val
+FROM ( SELECT 1 as val )
+; -- { serverError CYCLIC_ALIASES, TOO_DEEP_RECURSION }
+
+
+SELECT
+    val,
+    val + 1 as prev,
+    val + prev as val2
+FROM ( SELECT 1 as val )
+;