MDEV-26459 Assertion `block_size <= 0xFFFFFFFFL' failed in calculate_block_sizes for 10.7 only

Limit default allocation block in tree of Unique class

Author: sanja-byelkin

mysql-test/main/delete.result

@@ -557,3 +557,14 @@ having t3.a > any (select t2.b from t2
 where t2.b*10 < sum(t3.b)));
 drop table t1,t2,t3;
 End of 10.4 tests
+#
+# MDEV-26459: Assertion `block_size <= 0xFFFFFFFFL' failed
+# in calculate_block_sizes for 10.7 only
+#
+SET @sort_buffer_size_save= @@sort_buffer_size;
+SET sort_buffer_size=1125899906842624;
+CREATE TABLE t1 (a INT,b CHAR,KEY(a,b));
+DELETE a1 FROM t1 AS a1,t1 AS a2 WHERE a1.a=a2.a;
+DROP TABLE t1;
+SET sort_buffer_size= @sort_buffer_size_save;
+# End of 10.11 tests

mysql-test/main/delete.test

@@ -625,3 +625,17 @@ update t1 set t1.a=t1.a+10
 drop table t1,t2,t3;
 
 --echo End of 10.4 tests
+
+--echo #
+--echo # MDEV-26459: Assertion `block_size <= 0xFFFFFFFFL' failed
+--echo # in calculate_block_sizes for 10.7 only
+--echo #
+
+SET @sort_buffer_size_save= @@sort_buffer_size;
+SET sort_buffer_size=1125899906842624;
+CREATE TABLE t1 (a INT,b CHAR,KEY(a,b));
+DELETE a1 FROM t1 AS a1,t1 AS a2 WHERE a1.a=a2.a;
+DROP TABLE t1;
+SET sort_buffer_size= @sort_buffer_size_save;
+
+--echo # End of 10.11 tests

sql/uniques.cc

@@ -90,8 +90,10 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg,
   if (min_dupl_count_arg)
     full_size+= sizeof(element_count);
   with_counters= MY_TEST(min_dupl_count_arg);
-  init_tree(&tree, (max_in_memory_size / 16), 0, size, comp_func,
-            NULL, comp_func_fixed_arg, MYF(MY_THREAD_SPECIFIC));
+
+  init_tree(&tree, MY_MIN(max_in_memory_size / 16, UINT_MAX32),
+            0, size, comp_func, NULL, comp_func_fixed_arg,
+            MYF(MY_THREAD_SPECIFIC));
   /* If the following fail's the next add will also fail */
   my_init_dynamic_array(PSI_INSTRUMENT_ME, &file_ptrs, sizeof(Merge_chunk), 16,
                         16, MYF(MY_THREAD_SPECIFIC));