MDEV-28054 Various crashes upon INSERT/UPDATE after changing Aria set…

…tings

The cause of the crash was that test was setting
aria_sort_buffer_size to MAX_LONG_LONG, which caused an overflow in
my_malloc() when trying to allocate the buffer + 8 bytes.

Fixed by reducing max size of sort_buffer for Aria and MyISAM

Other things:
- Added code in maria_repair_parallell() to not allocate a big sort buffer
  for small files.
- Updated size of minumim sort buffer in Aria

mysql-test/suite/maria/maria.result

@@ -2814,7 +2814,7 @@ DROP TABLE t1;
 #             cardinalities=1
 #
 SET aria_repair_threads=2;
-SET aria_sort_buffer_size=8192;
+SET aria_sort_buffer_size=16384;
 CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
 Warnings:
 Note	1831	Duplicate index `a_2`. This is deprecated and will be disallowed in a future release
@@ -2839,19 +2839,8 @@ SET aria_repair_threads=@@global.aria_repair_threads;
 #             low myisam_sort_buffer_size
 #
 CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
-INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
-(6,'0'),(7,'0');
-INSERT INTO t1 SELECT a+10,b FROM t1;
-INSERT INTO t1 SELECT a+20,b FROM t1;
-INSERT INTO t1 SELECT a+40,b FROM t1;
-INSERT INTO t1 SELECT a+80,b FROM t1;
-INSERT INTO t1 SELECT a+160,b FROM t1;
-INSERT INTO t1 SELECT a+320,b FROM t1;
-INSERT INTO t1 SELECT a+640,b FROM t1;
-INSERT INTO t1 SELECT a+1280,b FROM t1;
-INSERT INTO t1 SELECT a+2560,b FROM t1;
-INSERT INTO t1 SELECT a+5120,b FROM t1;
-SET aria_sort_buffer_size=4096;
+INSERT INTO t1 select seq,'0' from seq_1_to_65536;
+SET aria_sort_buffer_size=16384;
 REPAIR TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	repair	error	aria_sort_buffer_size is too small. X

mysql-test/suite/maria/maria.test

@@ -5,6 +5,7 @@
 
 -- source include/have_maria.inc
 -- source include/have_partition.inc
+-- source include/have_sequence.inc
 
 call mtr.add_suppression("Can't find record in '.*'");
 
@@ -2034,7 +2035,7 @@ DROP TABLE t1;
 --echo #             cardinalities=1
 --echo #
 SET aria_repair_threads=2;
-SET aria_sort_buffer_size=8192;
+SET aria_sort_buffer_size=16384;
 CREATE TABLE t1(a CHAR(255), KEY(a), KEY(a), KEY(a));
 INSERT INTO t1 VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(0),(1),(2),(3);
 --replace_regex /Current aria_sort_buffer_size.*/X/
@@ -2050,19 +2051,8 @@ SET aria_repair_threads=@@global.aria_repair_threads;
 --echo #             low myisam_sort_buffer_size
 --echo #
 CREATE TABLE t1(a INT, b CHAR(10), KEY(a), KEY(b));
-INSERT INTO t1 VALUES(1,'0'),(2,'0'),(3,'0'),(4,'0'),(5,'0'),
-                     (6,'0'),(7,'0');
-INSERT INTO t1 SELECT a+10,b FROM t1;
-INSERT INTO t1 SELECT a+20,b FROM t1;
-INSERT INTO t1 SELECT a+40,b FROM t1;
-INSERT INTO t1 SELECT a+80,b FROM t1;
-INSERT INTO t1 SELECT a+160,b FROM t1;
-INSERT INTO t1 SELECT a+320,b FROM t1;
-INSERT INTO t1 SELECT a+640,b FROM t1;
-INSERT INTO t1 SELECT a+1280,b FROM t1;
-INSERT INTO t1 SELECT a+2560,b FROM t1;
-INSERT INTO t1 SELECT a+5120,b FROM t1;
-SET aria_sort_buffer_size=4096;
+INSERT INTO t1 select seq,'0' from seq_1_to_65536;
+SET aria_sort_buffer_size=16384;
 --replace_regex /Current aria_sort_buffer_size.*/X/
 REPAIR TABLE t1;
 CHECK TABLE t1;

mysql-test/suite/maria/repair-big-sort.result

@@ -0,0 +1,81 @@
+SET sql_mode='';
+CREATE TEMPORARY TABLE t1 (a tinyINT,b CHAR(1)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 VALUES (1,1),(3,3),(2,2);
+SET SESSION tmp_table_size=True;
+Warnings:
+Warning	1292	Truncated incorrect tmp_table_size value: '1'
+CREATE TABLE t2 (c INT, d DATE) ENGINE=InnoDB PARTITION BY RANGE (YEAR (d)) SUBPARTITION BY HASH (TO_DAYS (d)) (PARTITION p0 VALUES LESS THAN (1990) (SUBPARTITION s0, SUBPARTITION s1), PARTITION p1 VALUES LESS THAN MAXVALUE (SUBPARTITION s4, SUBPARTITION s5));
+SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT);
+Warnings:
+Note	1105	Cast to unsigned converted negative integer to it's positive complement
+Note	1105	Cast to unsigned converted negative integer to it's positive complement
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '18446744073709551615'
+INSERT INTO t1 SELECT '', SEQ FROM seq_1_to_258;
+Warnings:
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 1
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 2
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 3
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 4
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 5
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 6
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 7
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 8
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 9
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 10
+Warning	1265	Data truncated for column 'b' at row 10
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 11
+Warning	1265	Data truncated for column 'b' at row 11
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 12
+Warning	1265	Data truncated for column 'b' at row 12
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 13
+Warning	1265	Data truncated for column 'b' at row 13
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 14
+Warning	1265	Data truncated for column 'b' at row 14
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 15
+Warning	1265	Data truncated for column 'b' at row 15
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 16
+Warning	1265	Data truncated for column 'b' at row 16
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 17
+Warning	1265	Data truncated for column 'b' at row 17
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 18
+Warning	1265	Data truncated for column 'b' at row 18
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 19
+Warning	1265	Data truncated for column 'b' at row 19
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 20
+Warning	1265	Data truncated for column 'b' at row 20
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 21
+Warning	1265	Data truncated for column 'b' at row 21
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 22
+Warning	1265	Data truncated for column 'b' at row 22
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 23
+Warning	1265	Data truncated for column 'b' at row 23
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 24
+Warning	1265	Data truncated for column 'b' at row 24
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 25
+Warning	1265	Data truncated for column 'b' at row 25
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 26
+Warning	1265	Data truncated for column 'b' at row 26
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 27
+Warning	1265	Data truncated for column 'b' at row 27
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 28
+Warning	1265	Data truncated for column 'b' at row 28
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 29
+Warning	1265	Data truncated for column 'b' at row 29
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 30
+Warning	1265	Data truncated for column 'b' at row 30
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 31
+Warning	1265	Data truncated for column 'b' at row 31
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 32
+Warning	1265	Data truncated for column 'b' at row 32
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 33
+Warning	1265	Data truncated for column 'b' at row 33
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 34
+Warning	1265	Data truncated for column 'b' at row 34
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 35
+Warning	1265	Data truncated for column 'b' at row 35
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 36
+Warning	1265	Data truncated for column 'b' at row 36
+Warning	1366	Incorrect integer value: '' for column `test`.`t1`.`a` at row 37
+SET SESSION aria_repair_threads=4;
+UPDATE t1 SET a=( (SELECT MAX(a) FROM t1));
+drop table t1,t2;

mysql-test/suite/maria/repair-big-sort.test

@@ -0,0 +1,15 @@
+
+--source include/have_innodb.inc
+--source include/have_partition.inc
+--source include/have_sequence.inc
+
+SET sql_mode='';
+CREATE TEMPORARY TABLE t1 (a tinyINT,b CHAR(1)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 VALUES (1,1),(3,3),(2,2);
+SET SESSION tmp_table_size=True;
+CREATE TABLE t2 (c INT, d DATE) ENGINE=InnoDB PARTITION BY RANGE (YEAR (d)) SUBPARTITION BY HASH (TO_DAYS (d)) (PARTITION p0 VALUES LESS THAN (1990) (SUBPARTITION s0, SUBPARTITION s1), PARTITION p1 VALUES LESS THAN MAXVALUE (SUBPARTITION s4, SUBPARTITION s5));
+SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT);
+INSERT INTO t1 SELECT '', SEQ FROM seq_1_to_258;
+SET SESSION aria_repair_threads=4;
+UPDATE t1 SET a=( (SELECT MAX(a) FROM t1));
+drop table t1,t2;

mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result

@@ -22,13 +22,13 @@ Warnings:
 Warning	1292	Truncated incorrect aria_sort_buffer_size value: '10'
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-4096
+16376
 set session aria_sort_buffer_size=10;
 Warnings:
 Warning	1292	Truncated incorrect aria_sort_buffer_size value: '10'
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-4096
+16376
 set global aria_sort_buffer_size=1.1;
 ERROR 42000: Incorrect argument type to variable 'aria_sort_buffer_size'
 set session aria_sort_buffer_size=1e1;
@@ -40,9 +40,9 @@ Warnings:
 Warning	1292	Truncated incorrect aria_sort_buffer_size value: '0'
 select @@global.aria_sort_buffer_size;
 @@global.aria_sort_buffer_size
-4096
+16376
 set session aria_sort_buffer_size=cast(-1 as unsigned int);
 select @@session.aria_sort_buffer_size;
 @@session.aria_sort_buffer_size
-18446744073709551615
+1152921504606846975
 SET @@global.aria_sort_buffer_size = @start_global_value;

mysql-test/suite/sys_vars/r/sysvars_aria.result

@@ -223,8 +223,8 @@ DEFAULT_VALUE	268434432
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
-NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MIN_VALUE	16376
+NUMERIC_MAX_VALUE	1152921504606846975
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff

@@ -97,9 +97,9 @@
 @@ -207,7 +207,7 @@ VARIABLE_SCOPE	SESSION
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
- NUMERIC_MIN_VALUE	4096
+ NUMERIC_MIN_VALUE	16376
 -NUMERIC_MAX_VALUE	18446744073709551615
-+NUMERIC_MAX_VALUE	4294967295
++NUMERIC_MAX_VALUE	268435455
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO
@@ -684,8 +684,8 @@
  VARIABLE_TYPE	BIGINT UNSIGNED
  VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE
  NUMERIC_MIN_VALUE	4096
--NUMERIC_MAX_VALUE	18446744073709551615
-+NUMERIC_MAX_VALUE	4294967295
+-NUMERIC_MAX_VALUE	1152921504606846975
++NUMERIC_MAX_VALUE	268435455
  NUMERIC_BLOCK_SIZE	1
  ENUM_VALUE_LIST	NULL
  READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_server_embedded.result

@@ -216,8 +216,8 @@ VARIABLE_NAME	ARIA_SORT_BUFFER_SIZE
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
-NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MIN_VALUE	16376
+NUMERIC_MAX_VALUE	1152921504606846975
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
@@ -2127,7 +2127,7 @@ VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE
 NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	1152921504606846975
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO

mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result

@@ -216,8 +216,8 @@ VARIABLE_NAME	ARIA_SORT_BUFFER_SIZE
 VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.
-NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MIN_VALUE	16376
+NUMERIC_MAX_VALUE	1152921504606846975
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
@@ -2287,7 +2287,7 @@ VARIABLE_SCOPE	SESSION
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE
 NUMERIC_MIN_VALUE	4096
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	1152921504606846975
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO

storage/maria/ha_maria.cc

@@ -269,7 +269,7 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG,
 static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG,
        "The buffer that is allocated when sorting the index when doing a "
        "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL,
-       SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1);
+       SORT_BUFFER_INIT, MARIA_MIN_SORT_MEMORY, SIZE_T_MAX/16, 1);
 
 static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG,
        "Specifies how Aria index statistics collection code should treat "

storage/maria/ma_check.c

@@ -3903,6 +3903,16 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info,
     {
       sort_param.key_read=  sort_key_read;
       sort_param.key_write= sort_key_write;
+
+      /*
+        Limit usage of sort memory
+        We assume we don't need more memory than data file length * 2
+        (There is a pointer overhead for each key, but this is hard to
+        estimae as we cannot be sure how many records we have in file to
+        be repaired).
+      */
+      set_if_smaller(param->sort_buffer_length, sort_info.filelength*2);
+      set_if_bigger(param->sort_buffer_length, MARIA_MIN_SORT_MEMORY);
     }
 
     if (sort_info.new_info->s->data_file_type == BLOCK_RECORD)
@@ -4505,6 +4515,16 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
     DBUG_PRINT("io_cache_share", ("thread: %u  read_cache: %p",
                                   i, &sort_param[i].read_cache));
 
+    /*
+      Limit usage of sort memory
+      We assume we don't need more memory than data file length * 2
+      (There is a pointer overhead for each key, but this is hard to
+      estimae as we cannot be sure how many records we have in file to
+      be repaired).
+     */
+    set_if_smaller(param->sort_buffer_length, sort_info.filelength*2);
+    set_if_bigger(param->sort_buffer_length, MARIA_MIN_SORT_MEMORY);
+
     /*
       two approaches: the same amount of memory for each thread
       or the memory for the same number of keys for each thread...
@@ -4517,6 +4537,8 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info,
 #else
       param->sort_buffer_length*sort_param[i].key_length/total_key_length;
 #endif
+    set_if_bigger(sort_param[i].sortbuff_size, MARIA_MIN_SORT_MEMORY);
+
     if (mysql_thread_create(key_thread_find_all_keys,
                             &sort_param[i].thr, &thr_attr,
 	                    _ma_thr_find_all_keys, (void *) (sort_param+i)))