From eacefbca3596fa9cb853272265855d4efafd5f24 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 1 Feb 2021 18:46:34 +0200 Subject: [PATCH 1/3] MDEV-24750 Various corruptions caused by Aria subsystem... The test case was setting aria_sort_buffer_size to MAX_ULONGLONG-1 which was not handled gracefully by my_malloc() or safemalloc(). Fixed by ensuring that the malloc functions returns 0 if the size is too big. I also added some protection to Aria repair: - Limit sort_buffer_size to 16G (after that a bigger sort buffer will not help that much anyway) - Limit sort_buffer_size also according to sort file size. This will help by not allocating less memory if someone sets the buffer size too high. --- .../suite/maria/aria_sort_buffer.result | 11 ++++ mysql-test/suite/maria/aria_sort_buffer.test | 15 +++++ .../r/aria_sort_buffer_size_basic,32bit.rdiff | 9 +++ .../r/aria_sort_buffer_size_basic.result | 2 +- .../suite/sys_vars/r/sysvars_aria,32bit.rdiff | 8 +-- .../suite/sys_vars/r/sysvars_aria.result | 2 +- .../r/sysvars_server_embedded,32bit.rdiff | 58 ++++++++----------- .../sys_vars/r/sysvars_server_embedded.result | 2 +- .../r/sysvars_server_notembedded,32bit.rdiff | 4 +- .../r/sysvars_server_notembedded.result | 2 +- .../t/aria_sort_buffer_size_basic.test | 1 + mysys/my_malloc.c | 2 + mysys/safemalloc.c | 5 +- storage/maria/ha_maria.cc | 5 +- storage/maria/ma_check.c | 16 +++++ 15 files changed, 94 insertions(+), 48 deletions(-) create mode 100644 mysql-test/suite/maria/aria_sort_buffer.result create mode 100644 mysql-test/suite/maria/aria_sort_buffer.test create mode 100644 mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff diff --git a/mysql-test/suite/maria/aria_sort_buffer.result b/mysql-test/suite/maria/aria_sort_buffer.result new file mode 100644 index 0000000000000..5db9b9d7d44d7 --- /dev/null +++ b/mysql-test/suite/maria/aria_sort_buffer.result @@ -0,0 +1,11 @@ +SET SESSION aria_repair_threads=128; +SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT); +Warnings: +Note 1105 Cast to unsigned converted negative integer to it's positive complement +Note 1105 Cast to unsigned converted negative integer to it's positive complement +Warning 1292 Truncated incorrect aria_sort_buffer_size value: '18446744073709551615' +SET SESSION tmp_table_size=65535; +CREATE TABLE t1 (a VARCHAR(255)); +insert into t1 (a) select seq from seq_1_to_1000; +UPDATE t1 SET a=( (SELECT MAX(a) FROM t1)); +DROP TABLE t1; diff --git a/mysql-test/suite/maria/aria_sort_buffer.test b/mysql-test/suite/maria/aria_sort_buffer.test new file mode 100644 index 0000000000000..190daa0fa109f --- /dev/null +++ b/mysql-test/suite/maria/aria_sort_buffer.test @@ -0,0 +1,15 @@ +--source include/have_sequence.inc + +# +# +# MDEV-24750 Various corruptions caused by Aria subsystem asking system call +# to overwrite memory that it does not own (InnoDB stacks) + +SET SESSION aria_repair_threads=128; +SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT); + +SET SESSION tmp_table_size=65535; +CREATE TABLE t1 (a VARCHAR(255)); +insert into t1 (a) select seq from seq_1_to_1000; +UPDATE t1 SET a=( (SELECT MAX(a) FROM t1)); +DROP TABLE t1; diff --git a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff new file mode 100644 index 0000000000000..c30b99f1f95d0 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff @@ -0,0 +1,9 @@ +--- suite/sys_vars/r/aria_sort_buffer_size_basic.result 2021-02-02 02:58:55.686921205 +0200 ++++ suite/sys_vars/r/aria_sort_buffer_size_basic.reject 2021-02-02 11:02:12.361178360 +0200 +@@ -44,5 +44,5 @@ + set session aria_sort_buffer_size=cast(-1 as unsigned int); + select @@session.aria_sort_buffer_size; + @@session.aria_sort_buffer_size +-9223372036854775807 ++2147483647 + SET @@global.aria_sort_buffer_size = @start_global_value; diff --git a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result index 56522566ec9c1..b2a87db3f8cc7 100644 --- a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result +++ b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result @@ -44,5 +44,5 @@ select @@global.aria_sort_buffer_size; set session aria_sort_buffer_size=cast(-1 as unsigned int); select @@session.aria_sort_buffer_size; @@session.aria_sort_buffer_size -18446744073709551615 +9223372036854775807 SET @@global.aria_sort_buffer_size = @start_global_value; diff --git a/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff index 17d04208aac71..f0cbbd874ee08 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff @@ -1,5 +1,5 @@ ---- sysvars_aria.result 2020-04-29 11:35:30.042243474 +0300 -+++ sysvars_aria,32bit.reject 2020-04-30 11:45:51.128740906 +0300 +--- suite/sys_vars/r/sysvars_aria.result 2021-02-02 02:58:55.686921205 +0200 ++++ suite/sys_vars/r/sysvars_aria,32bit.reject 2021-02-02 10:55:53.876791633 +0200 @@ -5,7 +5,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 8192 @@ -98,8 +98,8 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 9223372036854775807 ++NUMERIC_MAX_VALUE 2147483647 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_aria.result b/mysql-test/suite/sys_vars/r/sysvars_aria.result index 5b0ac74a8895d..8cec099d28c39 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_aria.result +++ b/mysql-test/suite/sys_vars/r/sysvars_aria.result @@ -212,7 +212,7 @@ VARIABLE_SCOPE SESSION VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff index 37cebcf70dc07..c8049d28c3597 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff @@ -1,5 +1,5 @@ ---- mysql-test/suite/sys_vars/r/sysvars_server_embedded.result -+++ mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +--- suite/sys_vars/r/sysvars_server_embedded.result 2021-02-02 14:07:21.616784062 +0200 ++++ suite/sys_vars/r/sysvars_server_embedded,32bit.reject 2021-02-02 18:56:53.727764717 +0200 @@ -35,7 +35,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME ARIA_BLOCK_SIZE @@ -98,8 +98,8 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 9223372036854775807 ++NUMERIC_MAX_VALUE 2147483647 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO @@ -335,15 +335,6 @@ VARIABLE_COMMENT Number of best matches to use for query expansion NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -1058,7 +1058,7 @@ - VARIABLE_TYPE BIGINT UNSIGNED - VARIABLE_COMMENT The maximum length of the result of function GROUP_CONCAT() - NUMERIC_MIN_VALUE 4 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 - NUMERIC_BLOCK_SIZE 1 - ENUM_VALUE_LIST NULL - READ_ONLY NO @@ -1185,7 +1185,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME HISTOGRAM_SIZE @@ -554,7 +545,7 @@ -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored) - NUMERIC_MIN_VALUE 4 + NUMERIC_MIN_VALUE 64 NUMERIC_MAX_VALUE 8388608 @@ -1955,7 +1955,7 @@ COMMAND_LINE_ARGUMENT REQUIRED @@ -713,7 +704,7 @@ VARIABLE_COMMENT If this is not 0, then mysqld will use this value to reserve file descriptors to use with setrlimit(). If this value is 0 or autoset then mysqld will reserve max_connections*5 or max_connections + table_cache*2 (whichever is larger) number of file descriptors NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -2235,10 +2235,10 @@ +@@ -2235,17 +2235,17 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_MAX_SEL_ARG_WEIGHT VARIABLE_SCOPE SESSION @@ -726,7 +717,6 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2245,7 +2245,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_PRUNE_LEVEL VARIABLE_SCOPE SESSION @@ -791,7 +781,7 @@ +VARIABLE_TYPE INT VARIABLE_COMMENT Size of the statement digest. Use 0 to disable, -1 for automated sizing. NUMERIC_MIN_VALUE -1 - NUMERIC_MAX_VALUE 200 + NUMERIC_MAX_VALUE 1048576 @@ -2345,7 +2345,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME PERFORMANCE_SCHEMA_EVENTS_STAGES_HISTORY_LONG_SIZE @@ -810,7 +800,7 @@ VARIABLE_COMMENT Number of rows per thread in EVENTS_STAGES_HISTORY. Use 0 to disable, -1 for automated sizing. NUMERIC_MIN_VALUE -1 NUMERIC_MAX_VALUE 1024 -@@ -2355,7 +2355,7 @@ +@@ -2365,7 +2365,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME PERFORMANCE_SCHEMA_EVENTS_STATEMENTS_HISTORY_LONG_SIZE VARIABLE_SCOPE GLOBAL @@ -828,7 +818,7 @@ VARIABLE_COMMENT Number of rows per thread in EVENTS_STATEMENTS_HISTORY. Use 0 to disable, -1 for automated sizing. NUMERIC_MIN_VALUE -1 NUMERIC_MAX_VALUE 1024 -@@ -2385,7 +2395,7 @@ +@@ -2385,7 +2385,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME PERFORMANCE_SCHEMA_EVENTS_TRANSACTIONS_HISTORY_LONG_SIZE VARIABLE_SCOPE GLOBAL @@ -1259,7 +1249,7 @@ VARIABLE_COMMENT When reading rows in sorted order after a sort, the rows are read through this buffer to avoid a disk seeks NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 2147483647 -@@ -2915,10 +2915,10 @@ +@@ -3015,10 +3015,10 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME ROWID_MERGE_BUFF_SIZE VARIABLE_SCOPE SESSION @@ -1272,7 +1262,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2955,7 +2955,7 @@ +@@ -3055,7 +3055,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME SERVER_ID VARIABLE_SCOPE SESSION @@ -1281,7 +1271,7 @@ VARIABLE_COMMENT Uniquely identifies the server instance in the community of replication partners NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -3025,7 +3025,7 @@ +@@ -3125,7 +3125,7 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME SLAVE_MAX_ALLOWED_PACKET VARIABLE_SCOPE GLOBAL @@ -1290,7 +1280,7 @@ VARIABLE_COMMENT The maximum packet length to sent successfully from the master to slave. NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 1073741824 -@@ -3035,7 +3035,7 @@ +@@ -3135,7 +3135,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME SLOW_LAUNCH_TIME VARIABLE_SCOPE GLOBAL @@ -1299,7 +1289,7 @@ VARIABLE_COMMENT If creating the thread takes longer than this value (in seconds), the Slow_launch_threads counter will be incremented NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 31536000 -@@ -3078,7 +3078,7 @@ +@@ -3178,7 +3178,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Each thread that needs to do a sort allocates a buffer of this size NUMERIC_MIN_VALUE 1024 @@ -1308,7 +1298,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -3285,7 +3285,7 @@ +@@ -3395,7 +3395,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME STORED_PROGRAM_CACHE VARIABLE_SCOPE GLOBAL @@ -1317,7 +1307,7 @@ VARIABLE_COMMENT The soft upper limit for number of cached stored routines for one connection. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 524288 -@@ -3365,7 +3365,7 @@ +@@ -3475,7 +3475,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME TABLE_DEFINITION_CACHE VARIABLE_SCOPE GLOBAL @@ -1326,7 +1316,7 @@ VARIABLE_COMMENT The number of cached table definitions NUMERIC_MIN_VALUE 400 NUMERIC_MAX_VALUE 2097152 -@@ -3375,7 +3375,7 @@ +@@ -3485,7 +3485,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME TABLE_OPEN_CACHE VARIABLE_SCOPE GLOBAL @@ -1335,7 +1325,7 @@ VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 1048576 -@@ -3435,7 +3435,7 @@ +@@ -3545,7 +3545,7 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME THREAD_CACHE_SIZE VARIABLE_SCOPE GLOBAL @@ -1344,7 +1334,7 @@ VARIABLE_COMMENT How many threads we should keep in a cache for reuse. These are freed after 5 minutes of idle time NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16384 -@@ -3518,7 +3518,7 @@ +@@ -3628,7 +3628,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Max size for data for an internal temporary on-disk MyISAM or Aria table. NUMERIC_MIN_VALUE 1024 @@ -1353,7 +1343,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -3528,7 +3528,7 @@ +@@ -3638,7 +3638,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size. NUMERIC_MIN_VALUE 0 @@ -1362,7 +1352,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -3538,14 +3538,14 @@ +@@ -3648,14 +3648,14 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. NUMERIC_MIN_VALUE 0 @@ -1379,7 +1369,7 @@ VARIABLE_COMMENT Allocation block size for transactions to be stored in binary log NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 134217728 -@@ -3555,7 +3555,7 @@ +@@ -3665,7 +3665,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME TRANSACTION_PREALLOC_SIZE VARIABLE_SCOPE SESSION @@ -1388,7 +1378,7 @@ VARIABLE_COMMENT Persistent buffer for transactions to be stored in binary log NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 134217728 -@@ -3695,7 +3695,7 @@ +@@ -3805,7 +3805,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME WAIT_TIMEOUT VARIABLE_SCOPE SESSION @@ -1397,7 +1387,7 @@ VARIABLE_COMMENT The number of seconds the server waits for activity on a connection before closing it NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 31536000 -@@ -3722,7 +3722,7 @@ +@@ -3832,7 +3832,7 @@ VARIABLE_NAME LOG_TC_SIZE GLOBAL_VALUE_ORIGIN AUTO VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index e0b4e3565bc0c..e4ab38f611de9 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -208,7 +208,7 @@ VARIABLE_SCOPE SESSION VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff index 59236c401f501..3ce1ed7b08273 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff @@ -98,8 +98,8 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 9223372036854775807 ++NUMERIC_MAX_VALUE 2147483647 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 107735142959b..7cdfdff7129a4 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -208,7 +208,7 @@ VARIABLE_SCOPE SESSION VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test b/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test index 818d66328f847..aa748778402bb 100644 --- a/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test +++ b/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test @@ -1,5 +1,6 @@ # ulong session --source include/have_maria.inc +--source include/word_size.inc SET @start_global_value = @@global.aria_sort_buffer_size; diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index 33880f023a9c2..4f378ba8603f6 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -78,6 +78,8 @@ void *my_malloc(PSI_memory_key key, size_t size, myf my_flags) /* Safety */ if (!size) size=1; + if (size > SIZE_T_MAX - 1024L*1024L*16L) /* Wrong call */ + return 0; /* We have to align size as we store MY_THREAD_SPECIFIC flag in the LSB */ size= ALIGN_SIZE(size); diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index d38c3bcfb6655..e2d59f8e82467 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -115,9 +115,10 @@ void *sf_malloc(size_t size, myf my_flags) init_done= 1; } - irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4); + if (size > SIZE_T_MAX - 1024L*1024L*16L) /* Wrong call */ + return 0; - if (!irem) + if (!(irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4))) return 0; /* we guarantee the alignment */ diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 4fc871040143f..5faba898ba984 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -268,8 +268,9 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, "The buffer that is allocated when sorting the index when doing a " - "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL, - SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1); + "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", + NULL, NULL, + SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX/2, 1); static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG, "Specifies how Aria index statistics collection code should treat " diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index a0c755b3a7d1b..c63b531934678 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -2359,6 +2359,8 @@ static int initialize_variables_for_repair(HA_CHECK *param, MARIA_SHARE *org_share) { MARIA_SHARE *share= info->s; + size_t tmp; + uint threads; /* We have to clear these variables first, as the cleanup-in-case-of-error @@ -2419,6 +2421,7 @@ static int initialize_variables_for_repair(HA_CHECK *param, /* calculate max_records */ sort_info->filelength= my_seek(info->dfile.file, 0L, MY_SEEK_END, MYF(0)); + param->max_progress= sort_info->filelength; if ((param->testflag & T_CREATE_MISSING_KEYS) || sort_info->org_data_file_type == COMPRESSED_RECORD) @@ -2431,6 +2434,19 @@ static int initialize_variables_for_repair(HA_CHECK *param, sort_info->max_records= (ha_rows) (sort_info->filelength / rec_length); } + /* We don't need a bigger sort buffer than file_length * 8 */ + threads= (param->testflag & T_REP_PARALLEL) ? (uint) share->base.keys : 1; + tmp= (size_t) MY_MIN(sort_info->filelength, + (my_off_t) (SIZE_T_MAX/10/threads)); + tmp= MY_MAX(tmp * 8 * threads, (size_t) 65536); /* Some margin */ + set_if_smaller(param->sort_buffer_length, tmp); + /* Protect against too big sort buffer length */ +#if SIZEOF_SIZE_T >= 8 + set_if_smaller(param->sort_buffer_length, 16LL*1024LL*1024LL*1024LL); +#else + set_if_smaller(param->sort_buffer_length, 1L*1024L*1024L*1024L); +#endif + /* Set up transaction handler so that we can see all rows */ if (param->max_trid == 0) { From 7ce643782ba32dda0c9e5cc98efa55d0ab48cb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sun, 7 Feb 2021 12:11:16 +0200 Subject: [PATCH 2/3] MDEV-23379 fixup: Remove dead PERFORMANCE_SCHEMA code --- storage/innobase/handler/ha_innodb.cc | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5e9a7d951e3c1..0aad1347435d7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -509,21 +509,14 @@ const struct _ft_vft_ext ft_vft_ext_result = {innobase_fts_get_version, /* All RWLOCK used in Innodb are SX-locks */ # define PSI_RWLOCK_KEY(n) {&n##_key, #n, PSI_RWLOCK_FLAG_SX} -/* Keys to register pthread mutexes/cond in the current file with +/* Keys to register pthread mutexes in the current file with performance schema */ -static mysql_pfs_key_t commit_cond_mutex_key; -static mysql_pfs_key_t commit_cond_key; static mysql_pfs_key_t pending_checkpoint_mutex_key; static PSI_mutex_info all_pthread_mutexes[] = { - PSI_KEY(commit_cond_mutex), PSI_KEY(pending_checkpoint_mutex), }; -static PSI_cond_info all_innodb_conds[] = { - PSI_KEY(commit_cond) -}; - # ifdef UNIV_PFS_MUTEX /* all_innodb_mutexes array contains mutexes that are performance schema instrumented if "UNIV_PFS_MUTEX" @@ -3917,9 +3910,6 @@ static int innodb_init(void* p) count = array_elements(all_innodb_files); mysql_file_register("innodb", all_innodb_files, count); # endif /* UNIV_PFS_IO */ - - count = array_elements(all_innodb_conds); - mysql_cond_register("innodb", all_innodb_conds, count); #endif /* HAVE_PSI_INTERFACE */ bool create_new_db = false; From 4f4a4cf9eb10fe29fb79a8321a5661d679659dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sun, 7 Feb 2021 12:19:24 +0200 Subject: [PATCH 3/3] MDEV-23399 fixup: Use plain pthread_cond The condition variables that were introduced in commit 7cffb5f6e8a231a041152447be8980ce35d2c9b8 (MDEV-23399) are never instrumented with PERFORMANCE_SCHEMA. Let us avoid the storage overhead and dead code. --- storage/innobase/buf/buf0buf.cc | 20 +++++++------- storage/innobase/buf/buf0dblwr.cc | 10 +++---- storage/innobase/buf/buf0flu.cc | 40 ++++++++++++++------------- storage/innobase/handler/ha_innodb.cc | 4 +-- storage/innobase/include/buf0buf.h | 6 ++-- storage/innobase/include/buf0dblwr.h | 4 +-- storage/innobase/log/log0log.cc | 4 +-- storage/innobase/srv/srv0start.cc | 8 +++--- 8 files changed, 49 insertions(+), 47 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index e740db82d3843..d531be43a8bd4 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1462,9 +1462,9 @@ bool buf_pool_t::create() mysql_mutex_init(flush_list_mutex_key, &flush_list_mutex, MY_MUTEX_INIT_FAST); - mysql_cond_init(0, &done_flush_LRU, nullptr); - mysql_cond_init(0, &done_flush_list, nullptr); - mysql_cond_init(0, &do_flush_list, nullptr); + pthread_cond_init(&done_flush_LRU, nullptr); + pthread_cond_init(&done_flush_list, nullptr); + pthread_cond_init(&do_flush_list, nullptr); try_LRU_scan= true; @@ -1525,9 +1525,9 @@ void buf_pool_t::close() allocator.deallocate_large_dodump(chunk->mem, &chunk->mem_pfx); } - mysql_cond_destroy(&done_flush_LRU); - mysql_cond_destroy(&done_flush_list); - mysql_cond_destroy(&do_flush_list); + pthread_cond_destroy(&done_flush_LRU); + pthread_cond_destroy(&done_flush_list); + pthread_cond_destroy(&do_flush_list); ut_free(chunks); chunks= nullptr; @@ -3694,8 +3694,8 @@ buf_page_create(fil_space_t *space, uint32_t offset, We must not hold buf_pool.mutex while waiting. */ timespec abstime; set_timespec_nsec(abstime, 1000000); - mysql_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex, - &abstime); + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex.m_mutex, + &abstime); } mtr_memo_push(mtr, block, MTR_MEMO_PAGE_X_FIX); } @@ -3719,8 +3719,8 @@ buf_page_create(fil_space_t *space, uint32_t offset, /* Wait for buf_page_write_complete() to release the I/O fix. */ timespec abstime; set_timespec_nsec(abstime, 1000000); - mysql_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex, - &abstime); + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex.m_mutex, + &abstime); goto loop; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 6a4a6ced074f5..afaa02e7ab176 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -58,7 +58,7 @@ inline void buf_dblwr_t::init(const byte *header) ut_ad(!batch_running); mysql_mutex_init(buf_dblwr_mutex_key, &mutex, nullptr); - mysql_cond_init(0, &cond, nullptr); + pthread_cond_init(&cond, nullptr); block1= page_id_t(0, mach_read_from_4(header + TRX_SYS_DOUBLEWRITE_BLOCK1)); block2= page_id_t(0, mach_read_from_4(header + TRX_SYS_DOUBLEWRITE_BLOCK2)); @@ -452,7 +452,7 @@ void buf_dblwr_t::close() ut_ad(!active_slot->first_free); ut_ad(!batch_running); - mysql_cond_destroy(&cond); + pthread_cond_destroy(&cond); for (int i= 0; i < 2; i++) { aligned_free(slots[i].write_buf); @@ -489,7 +489,7 @@ void buf_dblwr_t::write_completed() /* We can now reuse the doublewrite memory buffer: */ flush_slot->first_free= 0; batch_running= false; - mysql_cond_broadcast(&cond); + pthread_cond_broadcast(&cond); } mysql_mutex_unlock(&mutex); @@ -566,7 +566,7 @@ bool buf_dblwr_t::flush_buffered_writes(const ulint size) return false; if (!batch_running) break; - mysql_cond_wait(&cond, &mutex); + my_cond_wait(&cond, &mutex.m_mutex); } ut_ad(active_slot->reserved == active_slot->first_free); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 934feadb43715..21a01dbd2fe2f 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -138,7 +138,7 @@ inline void buf_pool_t::page_cleaner_wakeup() srv_max_buf_pool_modified_pct <= dirty_pct) { page_cleaner_is_idle= false; - mysql_cond_signal(&do_flush_list); + pthread_cond_signal(&do_flush_list); } } @@ -396,12 +396,12 @@ void buf_page_write_complete(const IORequest &request) { buf_LRU_free_page(bpage, true); if (!--buf_pool.n_flush_LRU) - mysql_cond_broadcast(&buf_pool.done_flush_LRU); + pthread_cond_broadcast(&buf_pool.done_flush_LRU); } else { if (!--buf_pool.n_flush_list) - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); } mysql_mutex_unlock(&buf_pool.mutex); @@ -1512,11 +1512,11 @@ void buf_flush_wait_batch_end(bool lru) tpool::tpool_wait_begin(); thd_wait_begin(nullptr, THD_WAIT_DISKIO); do - mysql_cond_wait(cond, &buf_pool.mutex); + my_cond_wait(cond, &buf_pool.mutex.m_mutex); while (n_flush); tpool::tpool_wait_end(); thd_wait_end(nullptr); - mysql_cond_broadcast(cond); + pthread_cond_broadcast(cond); } } @@ -1571,7 +1571,7 @@ ulint buf_flush_lists(ulint max_n, lsn_t lsn) if (running || (lsn && !UT_LIST_GET_LEN(buf_pool.flush_list))) { if (!running) - mysql_cond_broadcast(cond); + pthread_cond_broadcast(cond); mysql_mutex_unlock(&buf_pool.mutex); return 0; } @@ -1588,7 +1588,7 @@ ulint buf_flush_lists(ulint max_n, lsn_t lsn) mysql_mutex_unlock(&buf_pool.mutex); if (!n_flushing) - mysql_cond_broadcast(cond); + pthread_cond_broadcast(cond); buf_dblwr.flush_buffered_writes(); @@ -1749,14 +1749,15 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn) if (buf_flush_sync_lsn < sync_lsn) { buf_flush_sync_lsn= sync_lsn; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } do { tpool::tpool_wait_begin(); thd_wait_begin(nullptr, THD_WAIT_DISKIO); - mysql_cond_wait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex); + my_cond_wait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex); thd_wait_end(nullptr); tpool::tpool_wait_end(); @@ -1788,7 +1789,7 @@ void buf_flush_ahead(lsn_t lsn) if (buf_flush_sync_lsn < lsn) { buf_flush_sync_lsn= lsn; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); } @@ -1866,7 +1867,7 @@ ATTRIBUTE_COLD static void buf_flush_sync_for_checkpoint(lsn_t lsn) buf_flush_sync_lsn= 0; /* wake up buf_flush_wait_flushed() */ - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); lsn= std::max(lsn, target); @@ -2095,10 +2096,11 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) break; if (buf_pool.page_cleaner_idle()) - mysql_cond_wait(&buf_pool.do_flush_list, &buf_pool.flush_list_mutex); + my_cond_wait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex); else - mysql_cond_timedwait(&buf_pool.do_flush_list, &buf_pool.flush_list_mutex, - &abstime); + my_cond_timedwait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); set_timespec(abstime, 1); @@ -2120,7 +2122,7 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) { buf_flush_sync_lsn= 0; /* wake up buf_flush_wait_flushed() */ - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); } unemployed: buf_pool.page_cleaner_set_idle(true); @@ -2156,7 +2158,7 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) { n_flushed= buf_flush_lists(srv_max_io_capacity, lsn_limit); /* wake up buf_flush_wait_flushed() */ - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); goto try_checkpoint; } else if (!srv_adaptive_flushing) @@ -2233,7 +2235,7 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) if (UNIV_UNLIKELY(lsn_limit != 0)) goto furious_flush; buf_page_cleaner_is_active= false; - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); mysql_mutex_unlock(&buf_pool.flush_list_mutex); my_thread_end(); @@ -2287,8 +2289,8 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() set_timespec(abstime, INNODB_EXTEND_TIMEOUT_INTERVAL / 2); mysql_mutex_lock(&buf_pool.mutex); while (buf_pool.n_flush_list) - mysql_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex, - &abstime); + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex.m_mutex, + &abstime); mysql_mutex_unlock(&buf_pool.mutex); } } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 0aad1347435d7..e72b307854dd9 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -17204,7 +17204,7 @@ innodb_max_dirty_pages_pct_update( in_val); srv_max_dirty_pages_pct_lwm = in_val; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } srv_max_buf_pool_modified_pct = in_val; @@ -17238,7 +17238,7 @@ innodb_max_dirty_pages_pct_lwm_update( } srv_max_dirty_pages_pct_lwm = in_val; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } /*************************************************************//** diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index c4254f911dc84..12fb139f1bfa2 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1795,11 +1795,11 @@ class buf_pool_t /** Number of pending LRU flush. */ Atomic_counter n_flush_LRU; /** broadcast when n_flush_LRU reaches 0; protected by mutex */ - mysql_cond_t done_flush_LRU; + pthread_cond_t done_flush_LRU; /** Number of pending flush_list flush. */ Atomic_counter n_flush_list; /** broadcast when n_flush_list reaches 0; protected by mutex */ - mysql_cond_t done_flush_list; + pthread_cond_t done_flush_list; /** @name General fields */ /* @{ */ @@ -1948,7 +1948,7 @@ class buf_pool_t bool page_cleaner_is_idle; public: /** signalled to wake up the page_cleaner; protected by flush_list_mutex */ - mysql_cond_t do_flush_list; + pthread_cond_t do_flush_list; /** @return whether the page cleaner must sleep due to being idle */ bool page_cleaner_idle() const diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index e95f2844fdc21..fb9df55504c00 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -61,7 +61,7 @@ class buf_dblwr_t /** mutex protecting the data members below */ mysql_mutex_t mutex; /** condition variable for !batch_running */ - mysql_cond_t cond; + pthread_cond_t cond; /** whether a batch is being written from the doublewrite buffer */ bool batch_running; /** number of expected flush_buffered_writes_completed() calls */ @@ -160,7 +160,7 @@ class buf_dblwr_t { mysql_mutex_lock(&mutex); while (batch_running) - mysql_cond_wait(&cond, &mutex); + my_cond_wait(&cond, &mutex.m_mutex); mysql_mutex_unlock(&mutex); } } diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index be1a341d9e595..6dfeec0f92571 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2014, 2020, MariaDB Corporation. +Copyright (c) 2014, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1086,7 +1086,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() if (buf_page_cleaner_is_active) { thread_name = "page cleaner thread"; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); goto wait_suspend_loop; } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index b0bc3d3ede95c..a3678e20955de 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2017,9 +2017,9 @@ void innodb_shutdown() } mysql_mutex_lock(&buf_pool.flush_list_mutex); while (buf_page_cleaner_is_active) { - mysql_cond_signal(&buf_pool.do_flush_list); - mysql_cond_wait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex); + pthread_cond_signal(&buf_pool.do_flush_list); + my_cond_wait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); break;