MDEV-7912 multitable delete with wrongly set sort_buffer_size crashes in merge_buffers

Fixed overflow error that caused fewer bites to be allocated than
necessary on Windows 64 bit. This is due to ulong being 32 bit on
64 bit Windows and 64 bit on 64 bit Linux.

Author: cvicentiu

mysql-test/r/uniques_crash-7912.result

@@ -0,0 +1,18 @@
+set sql_mode="";
+drop table if exists t1,t2;
+Warnings:
+Note	1051	Unknown table 't1'
+Note	1051	Unknown table 't2'
+create table `t1` (`a` datetime not null) engine=InnoDB;
+create table `t2` (`a` int not null) engine=innodb;
+replace into t1 values (),();
+Warnings:
+Warning	1364	Field 'a' doesn't have a default value
+insert into t2 values(0);
+set session sort_buffer_size = 1024*1024*1024*1024;
+SET session debug_dbug= '+d,make_merge_buff_alloc_fail';
+delete d2 from t2 as d1, t1 as d2 where d1.a <=> d2.a;
+ERROR HY000: Out of memory (Needed 2 bytes)
+SET SESSION debug_dbug=DEFAULT;
+drop table t2;
+drop table t1;

mysql-test/t/uniques_crash-7912.test

@@ -0,0 +1,18 @@
+# MDEV-7912
+# multitable delete with wrongly set sort_buffer_size crashes in merge_buffers
+--source include/have_innodb.inc
+set sql_mode="";
+drop table if exists t1,t2;
+create table `t1` (`a` datetime not null) engine=InnoDB;
+create table `t2` (`a` int not null) engine=innodb;
+
+replace into t1 values (),();
+insert into t2 values(0);
+set session sort_buffer_size = 1024*1024*1024*1024;
+SET session debug_dbug= '+d,make_merge_buff_alloc_fail';
+--error 5 #EE_OUTOFMEMORY
+delete d2 from t2 as d1, t1 as d2 where d1.a <=> d2.a;
+SET SESSION debug_dbug=DEFAULT;
+
+drop table t2;
+drop table t1;

sql/uniques.cc

@@ -97,7 +97,7 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg,
   max_elements= (ulong) (max_in_memory_size /
                          ALIGN_SIZE(sizeof(TREE_ELEMENT)+size));
   (void) open_cached_file(&file, mysql_tmpdir,TEMP_PREFIX, DISK_BUFFER_SIZE,
-		   MYF(MY_WME));
+                          MYF(MY_WME));
 }
 
 
@@ -607,8 +607,10 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg)
     return 1;
   if (flush_io_cache(&file) || reinit_io_cache(&file, READ_CACHE, 0L, 0, 0))
     return 1;
-  ulong buff_sz= (max_in_memory_size / full_size + 1) * full_size;
-  if (!(merge_buffer= (uchar *) my_malloc((ulong) buff_sz, MYF(0))))
+  size_t buff_sz= (max_in_memory_size / full_size + 1) * full_size;
+  DBUG_EXECUTE_IF("make_merge_buff_alloc_fail",
+                  DBUG_SET("+d,simulate_out_of_memory"););
+  if (!(merge_buffer = (uchar *)my_malloc(buff_sz, MYF(MY_WME))))
     return 1;
   if (buff_sz < (ulong) (full_size * (file_ptrs.elements + 1)))
     res= merge(table, merge_buffer, buff_sz >= full_size * MERGEBUFF2) ;
@@ -737,9 +739,10 @@ bool Unique::get(TABLE *table)
   /* Not enough memory; Save the result to file && free memory used by tree */
   if (flush())
     return 1;
-  
-  ulong buff_sz= (max_in_memory_size / full_size + 1) * full_size;
-  if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(0))))
+  DBUG_EXECUTE_IF("make_merge_buff_alloc_fail",
+                  DBUG_SET("+d,simulate_out_of_memory"););
+  size_t buff_sz= (max_in_memory_size / full_size + 1) * full_size;
+  if (!(sort_buffer= (uchar*) my_malloc(buff_sz, MYF(MY_WME))))
     return 1;
 
   if (merge(table, sort_buffer, FALSE))