diff --git a/mysql-test/suite/maria/aria_sort_buffer.result b/mysql-test/suite/maria/aria_sort_buffer.result new file mode 100644 index 0000000000000..5db9b9d7d44d7 --- /dev/null +++ b/mysql-test/suite/maria/aria_sort_buffer.result @@ -0,0 +1,11 @@ +SET SESSION aria_repair_threads=128; +SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT); +Warnings: +Note 1105 Cast to unsigned converted negative integer to it's positive complement +Note 1105 Cast to unsigned converted negative integer to it's positive complement +Warning 1292 Truncated incorrect aria_sort_buffer_size value: '18446744073709551615' +SET SESSION tmp_table_size=65535; +CREATE TABLE t1 (a VARCHAR(255)); +insert into t1 (a) select seq from seq_1_to_1000; +UPDATE t1 SET a=( (SELECT MAX(a) FROM t1)); +DROP TABLE t1; diff --git a/mysql-test/suite/maria/aria_sort_buffer.test b/mysql-test/suite/maria/aria_sort_buffer.test new file mode 100644 index 0000000000000..190daa0fa109f --- /dev/null +++ b/mysql-test/suite/maria/aria_sort_buffer.test @@ -0,0 +1,15 @@ +--source include/have_sequence.inc + +# +# +# MDEV-24750 Various corruptions caused by Aria subsystem asking system call +# to overwrite memory that it does not own (InnoDB stacks) + +SET SESSION aria_repair_threads=128; +SET SESSION aria_sort_buffer_size=CAST(-1 AS UNSIGNED INT); + +SET SESSION tmp_table_size=65535; +CREATE TABLE t1 (a VARCHAR(255)); +insert into t1 (a) select seq from seq_1_to_1000; +UPDATE t1 SET a=( (SELECT MAX(a) FROM t1)); +DROP TABLE t1; diff --git a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff new file mode 100644 index 0000000000000..c30b99f1f95d0 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic,32bit.rdiff @@ -0,0 +1,9 @@ +--- suite/sys_vars/r/aria_sort_buffer_size_basic.result 2021-02-02 02:58:55.686921205 +0200 ++++ suite/sys_vars/r/aria_sort_buffer_size_basic.reject 2021-02-02 11:02:12.361178360 +0200 +@@ -44,5 +44,5 @@ + set session aria_sort_buffer_size=cast(-1 as unsigned int); + select @@session.aria_sort_buffer_size; + @@session.aria_sort_buffer_size +-9223372036854775807 ++2147483647 + SET @@global.aria_sort_buffer_size = @start_global_value; diff --git a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result index 56522566ec9c1..b2a87db3f8cc7 100644 --- a/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result +++ b/mysql-test/suite/sys_vars/r/aria_sort_buffer_size_basic.result @@ -44,5 +44,5 @@ select @@global.aria_sort_buffer_size; set session aria_sort_buffer_size=cast(-1 as unsigned int); select @@session.aria_sort_buffer_size; @@session.aria_sort_buffer_size -18446744073709551615 +9223372036854775807 SET @@global.aria_sort_buffer_size = @start_global_value; diff --git a/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff index 17d04208aac71..f0cbbd874ee08 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_aria,32bit.rdiff @@ -1,5 +1,5 @@ ---- sysvars_aria.result 2020-04-29 11:35:30.042243474 +0300 -+++ sysvars_aria,32bit.reject 2020-04-30 11:45:51.128740906 +0300 +--- suite/sys_vars/r/sysvars_aria.result 2021-02-02 02:58:55.686921205 +0200 ++++ suite/sys_vars/r/sysvars_aria,32bit.reject 2021-02-02 10:55:53.876791633 +0200 @@ -5,7 +5,7 @@ SESSION_VALUE NULL DEFAULT_VALUE 8192 @@ -98,8 +98,8 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 9223372036854775807 ++NUMERIC_MAX_VALUE 2147483647 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_aria.result b/mysql-test/suite/sys_vars/r/sysvars_aria.result index 5b0ac74a8895d..8cec099d28c39 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_aria.result +++ b/mysql-test/suite/sys_vars/r/sysvars_aria.result @@ -212,7 +212,7 @@ VARIABLE_SCOPE SESSION VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff index 37cebcf70dc07..c8049d28c3597 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff @@ -1,5 +1,5 @@ ---- mysql-test/suite/sys_vars/r/sysvars_server_embedded.result -+++ mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +--- suite/sys_vars/r/sysvars_server_embedded.result 2021-02-02 14:07:21.616784062 +0200 ++++ suite/sys_vars/r/sysvars_server_embedded,32bit.reject 2021-02-02 18:56:53.727764717 +0200 @@ -35,7 +35,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME ARIA_BLOCK_SIZE @@ -98,8 +98,8 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 9223372036854775807 ++NUMERIC_MAX_VALUE 2147483647 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO @@ -335,15 +335,6 @@ VARIABLE_COMMENT Number of best matches to use for query expansion NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -1058,7 +1058,7 @@ - VARIABLE_TYPE BIGINT UNSIGNED - VARIABLE_COMMENT The maximum length of the result of function GROUP_CONCAT() - NUMERIC_MIN_VALUE 4 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 - NUMERIC_BLOCK_SIZE 1 - ENUM_VALUE_LIST NULL - READ_ONLY NO @@ -1185,7 +1185,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME HISTOGRAM_SIZE @@ -554,7 +545,7 @@ -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored) - NUMERIC_MIN_VALUE 4 + NUMERIC_MIN_VALUE 64 NUMERIC_MAX_VALUE 8388608 @@ -1955,7 +1955,7 @@ COMMAND_LINE_ARGUMENT REQUIRED @@ -713,7 +704,7 @@ VARIABLE_COMMENT If this is not 0, then mysqld will use this value to reserve file descriptors to use with setrlimit(). If this value is 0 or autoset then mysqld will reserve max_connections*5 or max_connections + table_cache*2 (whichever is larger) number of file descriptors NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -2235,10 +2235,10 @@ +@@ -2235,17 +2235,17 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_MAX_SEL_ARG_WEIGHT VARIABLE_SCOPE SESSION @@ -726,7 +717,6 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2245,7 +2245,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME OPTIMIZER_PRUNE_LEVEL VARIABLE_SCOPE SESSION @@ -791,7 +781,7 @@ +VARIABLE_TYPE INT VARIABLE_COMMENT Size of the statement digest. Use 0 to disable, -1 for automated sizing. NUMERIC_MIN_VALUE -1 - NUMERIC_MAX_VALUE 200 + NUMERIC_MAX_VALUE 1048576 @@ -2345,7 +2345,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME PERFORMANCE_SCHEMA_EVENTS_STAGES_HISTORY_LONG_SIZE @@ -810,7 +800,7 @@ VARIABLE_COMMENT Number of rows per thread in EVENTS_STAGES_HISTORY. Use 0 to disable, -1 for automated sizing. NUMERIC_MIN_VALUE -1 NUMERIC_MAX_VALUE 1024 -@@ -2355,7 +2355,7 @@ +@@ -2365,7 +2365,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME PERFORMANCE_SCHEMA_EVENTS_STATEMENTS_HISTORY_LONG_SIZE VARIABLE_SCOPE GLOBAL @@ -828,7 +818,7 @@ VARIABLE_COMMENT Number of rows per thread in EVENTS_STATEMENTS_HISTORY. Use 0 to disable, -1 for automated sizing. NUMERIC_MIN_VALUE -1 NUMERIC_MAX_VALUE 1024 -@@ -2385,7 +2395,7 @@ +@@ -2385,7 +2385,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME PERFORMANCE_SCHEMA_EVENTS_TRANSACTIONS_HISTORY_LONG_SIZE VARIABLE_SCOPE GLOBAL @@ -1259,7 +1249,7 @@ VARIABLE_COMMENT When reading rows in sorted order after a sort, the rows are read through this buffer to avoid a disk seeks NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 2147483647 -@@ -2915,10 +2915,10 @@ +@@ -3015,10 +3015,10 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME ROWID_MERGE_BUFF_SIZE VARIABLE_SCOPE SESSION @@ -1272,7 +1262,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2955,7 +2955,7 @@ +@@ -3055,7 +3055,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME SERVER_ID VARIABLE_SCOPE SESSION @@ -1281,7 +1271,7 @@ VARIABLE_COMMENT Uniquely identifies the server instance in the community of replication partners NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -3025,7 +3025,7 @@ +@@ -3125,7 +3125,7 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME SLAVE_MAX_ALLOWED_PACKET VARIABLE_SCOPE GLOBAL @@ -1290,7 +1280,7 @@ VARIABLE_COMMENT The maximum packet length to sent successfully from the master to slave. NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 1073741824 -@@ -3035,7 +3035,7 @@ +@@ -3135,7 +3135,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME SLOW_LAUNCH_TIME VARIABLE_SCOPE GLOBAL @@ -1299,7 +1289,7 @@ VARIABLE_COMMENT If creating the thread takes longer than this value (in seconds), the Slow_launch_threads counter will be incremented NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 31536000 -@@ -3078,7 +3078,7 @@ +@@ -3178,7 +3178,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Each thread that needs to do a sort allocates a buffer of this size NUMERIC_MIN_VALUE 1024 @@ -1308,7 +1298,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -3285,7 +3285,7 @@ +@@ -3395,7 +3395,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME STORED_PROGRAM_CACHE VARIABLE_SCOPE GLOBAL @@ -1317,7 +1307,7 @@ VARIABLE_COMMENT The soft upper limit for number of cached stored routines for one connection. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 524288 -@@ -3365,7 +3365,7 @@ +@@ -3475,7 +3475,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME TABLE_DEFINITION_CACHE VARIABLE_SCOPE GLOBAL @@ -1326,7 +1316,7 @@ VARIABLE_COMMENT The number of cached table definitions NUMERIC_MIN_VALUE 400 NUMERIC_MAX_VALUE 2097152 -@@ -3375,7 +3375,7 @@ +@@ -3485,7 +3485,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME TABLE_OPEN_CACHE VARIABLE_SCOPE GLOBAL @@ -1335,7 +1325,7 @@ VARIABLE_COMMENT The number of cached open tables NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 1048576 -@@ -3435,7 +3435,7 @@ +@@ -3545,7 +3545,7 @@ COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME THREAD_CACHE_SIZE VARIABLE_SCOPE GLOBAL @@ -1344,7 +1334,7 @@ VARIABLE_COMMENT How many threads we should keep in a cache for reuse. These are freed after 5 minutes of idle time NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16384 -@@ -3518,7 +3518,7 @@ +@@ -3628,7 +3628,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Max size for data for an internal temporary on-disk MyISAM or Aria table. NUMERIC_MIN_VALUE 1024 @@ -1353,7 +1343,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -3528,7 +3528,7 @@ +@@ -3638,7 +3638,7 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. Same as tmp_table_size. NUMERIC_MIN_VALUE 0 @@ -1362,7 +1352,7 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -3538,14 +3538,14 @@ +@@ -3648,14 +3648,14 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Alias for tmp_memory_table_size. If an internal in-memory temporary table exceeds this size, MariaDB will automatically convert it to an on-disk MyISAM or Aria table. NUMERIC_MIN_VALUE 0 @@ -1379,7 +1369,7 @@ VARIABLE_COMMENT Allocation block size for transactions to be stored in binary log NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 134217728 -@@ -3555,7 +3555,7 @@ +@@ -3665,7 +3665,7 @@ COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME TRANSACTION_PREALLOC_SIZE VARIABLE_SCOPE SESSION @@ -1388,7 +1378,7 @@ VARIABLE_COMMENT Persistent buffer for transactions to be stored in binary log NUMERIC_MIN_VALUE 1024 NUMERIC_MAX_VALUE 134217728 -@@ -3695,7 +3695,7 @@ +@@ -3805,7 +3805,7 @@ COMMAND_LINE_ARGUMENT NULL VARIABLE_NAME WAIT_TIMEOUT VARIABLE_SCOPE SESSION @@ -1397,7 +1387,7 @@ VARIABLE_COMMENT The number of seconds the server waits for activity on a connection before closing it NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 31536000 -@@ -3722,7 +3722,7 @@ +@@ -3832,7 +3832,7 @@ VARIABLE_NAME LOG_TC_SIZE GLOBAL_VALUE_ORIGIN AUTO VARIABLE_SCOPE GLOBAL diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index e0b4e3565bc0c..e4ab38f611de9 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -208,7 +208,7 @@ VARIABLE_SCOPE SESSION VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff index 59236c401f501..3ce1ed7b08273 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff @@ -98,8 +98,8 @@ VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 --NUMERIC_MAX_VALUE 18446744073709551615 -+NUMERIC_MAX_VALUE 4294967295 +-NUMERIC_MAX_VALUE 9223372036854775807 ++NUMERIC_MAX_VALUE 2147483647 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 107735142959b..7cdfdff7129a4 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -208,7 +208,7 @@ VARIABLE_SCOPE SESSION VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT The buffer that is allocated when sorting the index when doing a REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE. NUMERIC_MIN_VALUE 4096 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 9223372036854775807 NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test b/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test index 818d66328f847..aa748778402bb 100644 --- a/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test +++ b/mysql-test/suite/sys_vars/t/aria_sort_buffer_size_basic.test @@ -1,5 +1,6 @@ # ulong session --source include/have_maria.inc +--source include/word_size.inc SET @start_global_value = @@global.aria_sort_buffer_size; diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index 33880f023a9c2..4f378ba8603f6 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -78,6 +78,8 @@ void *my_malloc(PSI_memory_key key, size_t size, myf my_flags) /* Safety */ if (!size) size=1; + if (size > SIZE_T_MAX - 1024L*1024L*16L) /* Wrong call */ + return 0; /* We have to align size as we store MY_THREAD_SPECIFIC flag in the LSB */ size= ALIGN_SIZE(size); diff --git a/mysys/safemalloc.c b/mysys/safemalloc.c index d38c3bcfb6655..e2d59f8e82467 100644 --- a/mysys/safemalloc.c +++ b/mysys/safemalloc.c @@ -115,9 +115,10 @@ void *sf_malloc(size_t size, myf my_flags) init_done= 1; } - irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4); + if (size > SIZE_T_MAX - 1024L*1024L*16L) /* Wrong call */ + return 0; - if (!irem) + if (!(irem= (struct st_irem *) malloc (sizeof(struct st_irem) + size + 4))) return 0; /* we guarantee the alignment */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index fc9816d02a100..03b74f105a129 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1452,9 +1452,9 @@ bool buf_pool_t::create() mysql_mutex_init(flush_list_mutex_key, &flush_list_mutex, MY_MUTEX_INIT_FAST); - mysql_cond_init(0, &done_flush_LRU, nullptr); - mysql_cond_init(0, &done_flush_list, nullptr); - mysql_cond_init(0, &do_flush_list, nullptr); + pthread_cond_init(&done_flush_LRU, nullptr); + pthread_cond_init(&done_flush_list, nullptr); + pthread_cond_init(&do_flush_list, nullptr); try_LRU_scan= true; @@ -1515,9 +1515,9 @@ void buf_pool_t::close() allocator.deallocate_large_dodump(chunk->mem, &chunk->mem_pfx); } - mysql_cond_destroy(&done_flush_LRU); - mysql_cond_destroy(&done_flush_list); - mysql_cond_destroy(&do_flush_list); + pthread_cond_destroy(&done_flush_LRU); + pthread_cond_destroy(&done_flush_list); + pthread_cond_destroy(&do_flush_list); ut_free(chunks); chunks= nullptr; @@ -3543,8 +3543,8 @@ buf_page_create(fil_space_t *space, uint32_t offset, We must not hold buf_pool.mutex while waiting. */ timespec abstime; set_timespec_nsec(abstime, 1000000); - mysql_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex, - &abstime); + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex.m_mutex, + &abstime); } mtr_memo_push(mtr, block, MTR_MEMO_PAGE_X_FIX); } @@ -3568,8 +3568,8 @@ buf_page_create(fil_space_t *space, uint32_t offset, /* Wait for buf_page_write_complete() to release the I/O fix. */ timespec abstime; set_timespec_nsec(abstime, 1000000); - mysql_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex, - &abstime); + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex.m_mutex, + &abstime); goto loop; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index c13e41b600da9..6f61fd4b6044e 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,7 +55,7 @@ inline void buf_dblwr_t::init(const byte *header) ut_ad(!batch_running); mysql_mutex_init(buf_dblwr_mutex_key, &mutex, nullptr); - mysql_cond_init(0, &cond, nullptr); + pthread_cond_init(&cond, nullptr); block1= page_id_t(0, mach_read_from_4(header + TRX_SYS_DOUBLEWRITE_BLOCK1)); block2= page_id_t(0, mach_read_from_4(header + TRX_SYS_DOUBLEWRITE_BLOCK2)); @@ -445,7 +445,7 @@ void buf_dblwr_t::close() ut_ad(!active_slot->first_free); ut_ad(!batch_running); - mysql_cond_destroy(&cond); + pthread_cond_destroy(&cond); for (int i= 0; i < 2; i++) { aligned_free(slots[i].write_buf); @@ -482,7 +482,7 @@ void buf_dblwr_t::write_completed() /* We can now reuse the doublewrite memory buffer: */ flush_slot->first_free= 0; batch_running= false; - mysql_cond_broadcast(&cond); + pthread_cond_broadcast(&cond); } mysql_mutex_unlock(&mutex); @@ -559,7 +559,7 @@ bool buf_dblwr_t::flush_buffered_writes(const ulint size) return false; if (!batch_running) break; - mysql_cond_wait(&cond, &mutex); + my_cond_wait(&cond, &mutex.m_mutex); } ut_ad(active_slot->reserved == active_slot->first_free); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 76a26733358a2..bc2faa9e3ea50 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -138,7 +138,7 @@ inline void buf_pool_t::page_cleaner_wakeup() srv_max_buf_pool_modified_pct <= dirty_pct) { page_cleaner_is_idle= false; - mysql_cond_signal(&do_flush_list); + pthread_cond_signal(&do_flush_list); } } @@ -391,12 +391,12 @@ void buf_page_write_complete(const IORequest &request) { buf_LRU_free_page(bpage, true); if (!--buf_pool.n_flush_LRU) - mysql_cond_broadcast(&buf_pool.done_flush_LRU); + pthread_cond_broadcast(&buf_pool.done_flush_LRU); } else { if (!--buf_pool.n_flush_list) - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); } mysql_mutex_unlock(&buf_pool.mutex); @@ -1506,11 +1506,11 @@ void buf_flush_wait_batch_end(bool lru) tpool::tpool_wait_begin(); thd_wait_begin(nullptr, THD_WAIT_DISKIO); do - mysql_cond_wait(cond, &buf_pool.mutex); + my_cond_wait(cond, &buf_pool.mutex.m_mutex); while (n_flush); tpool::tpool_wait_end(); thd_wait_end(nullptr); - mysql_cond_broadcast(cond); + pthread_cond_broadcast(cond); } } @@ -1565,7 +1565,7 @@ ulint buf_flush_lists(ulint max_n, lsn_t lsn) if (running || (lsn && !UT_LIST_GET_LEN(buf_pool.flush_list))) { if (!running) - mysql_cond_broadcast(cond); + pthread_cond_broadcast(cond); mysql_mutex_unlock(&buf_pool.mutex); return 0; } @@ -1582,7 +1582,7 @@ ulint buf_flush_lists(ulint max_n, lsn_t lsn) mysql_mutex_unlock(&buf_pool.mutex); if (!n_flushing) - mysql_cond_broadcast(cond); + pthread_cond_broadcast(cond); buf_dblwr.flush_buffered_writes(); @@ -1743,14 +1743,15 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn) if (buf_flush_sync_lsn < sync_lsn) { buf_flush_sync_lsn= sync_lsn; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } do { tpool::tpool_wait_begin(); thd_wait_begin(nullptr, THD_WAIT_DISKIO); - mysql_cond_wait(&buf_pool.done_flush_list, &buf_pool.flush_list_mutex); + my_cond_wait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex); thd_wait_end(nullptr); tpool::tpool_wait_end(); @@ -1782,7 +1783,7 @@ void buf_flush_ahead(lsn_t lsn) if (buf_flush_sync_lsn < lsn) { buf_flush_sync_lsn= lsn; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); } @@ -1860,7 +1861,7 @@ ATTRIBUTE_COLD static void buf_flush_sync_for_checkpoint(lsn_t lsn) buf_flush_sync_lsn= 0; /* wake up buf_flush_wait_flushed() */ - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); lsn= std::max(lsn, target); @@ -2089,10 +2090,11 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) break; if (buf_pool.page_cleaner_idle()) - mysql_cond_wait(&buf_pool.do_flush_list, &buf_pool.flush_list_mutex); + my_cond_wait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex); else - mysql_cond_timedwait(&buf_pool.do_flush_list, &buf_pool.flush_list_mutex, - &abstime); + my_cond_timedwait(&buf_pool.do_flush_list, + &buf_pool.flush_list_mutex.m_mutex, &abstime); set_timespec(abstime, 1); @@ -2114,7 +2116,7 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) { buf_flush_sync_lsn= 0; /* wake up buf_flush_wait_flushed() */ - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); } unemployed: buf_pool.page_cleaner_set_idle(true); @@ -2150,7 +2152,7 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) { n_flushed= buf_flush_lists(srv_max_io_capacity, lsn_limit); /* wake up buf_flush_wait_flushed() */ - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); goto try_checkpoint; } else if (!srv_adaptive_flushing) @@ -2227,7 +2229,7 @@ static os_thread_ret_t DECLARE_THREAD(buf_flush_page_cleaner)(void*) if (UNIV_UNLIKELY(lsn_limit != 0)) goto furious_flush; buf_page_cleaner_is_active= false; - mysql_cond_broadcast(&buf_pool.done_flush_list); + pthread_cond_broadcast(&buf_pool.done_flush_list); mysql_mutex_unlock(&buf_pool.flush_list_mutex); my_thread_end(); @@ -2281,8 +2283,8 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() set_timespec(abstime, INNODB_EXTEND_TIMEOUT_INTERVAL / 2); mysql_mutex_lock(&buf_pool.mutex); while (buf_pool.n_flush_list) - mysql_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex, - &abstime); + my_cond_timedwait(&buf_pool.done_flush_list, &buf_pool.mutex.m_mutex, + &abstime); mysql_mutex_unlock(&buf_pool.mutex); } } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 033c062dc8ebf..64e802effaa8d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -491,21 +491,14 @@ const struct _ft_vft_ext ft_vft_ext_result = {innobase_fts_get_version, #ifdef HAVE_PSI_INTERFACE # define PSI_KEY(n) {&n##_key, #n, 0} -/** Keys to register pthread mutexes/cond in the current file with +/* Keys to register pthread mutexes in the current file with performance schema */ -static mysql_pfs_key_t commit_cond_mutex_key; -static mysql_pfs_key_t commit_cond_key; static mysql_pfs_key_t pending_checkpoint_mutex_key; static PSI_mutex_info all_pthread_mutexes[] = { - PSI_KEY(commit_cond_mutex), PSI_KEY(pending_checkpoint_mutex), }; -static PSI_cond_info all_innodb_conds[] = { - PSI_KEY(commit_cond) -}; - # ifdef UNIV_PFS_MUTEX mysql_pfs_key_t buf_pool_mutex_key; mysql_pfs_key_t dict_foreign_err_mutex_key; @@ -3697,9 +3690,6 @@ static int innodb_init(void* p) count = array_elements(all_innodb_files); mysql_file_register("innodb", all_innodb_files, count); # endif /* UNIV_PFS_IO */ - - count = array_elements(all_innodb_conds); - mysql_cond_register("innodb", all_innodb_conds, count); #endif /* HAVE_PSI_INTERFACE */ bool create_new_db = false; @@ -16654,7 +16644,7 @@ innodb_max_dirty_pages_pct_update( in_val); srv_max_dirty_pages_pct_lwm = in_val; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } srv_max_buf_pool_modified_pct = in_val; @@ -16688,7 +16678,7 @@ innodb_max_dirty_pages_pct_lwm_update( } srv_max_dirty_pages_pct_lwm = in_val; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); } /*************************************************************//** diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 0219ebfd74797..11b9d41b592aa 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1716,11 +1716,11 @@ class buf_pool_t /** Number of pending LRU flush. */ Atomic_counter n_flush_LRU; /** broadcast when n_flush_LRU reaches 0; protected by mutex */ - mysql_cond_t done_flush_LRU; + pthread_cond_t done_flush_LRU; /** Number of pending flush_list flush. */ Atomic_counter n_flush_list; /** broadcast when n_flush_list reaches 0; protected by mutex */ - mysql_cond_t done_flush_list; + pthread_cond_t done_flush_list; /** @name General fields */ /* @{ */ @@ -1869,7 +1869,7 @@ class buf_pool_t bool page_cleaner_is_idle; public: /** signalled to wake up the page_cleaner; protected by flush_list_mutex */ - mysql_cond_t do_flush_list; + pthread_cond_t do_flush_list; /** @return whether the page cleaner must sleep due to being idle */ bool page_cleaner_idle() const diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index e95f2844fdc21..fb9df55504c00 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -61,7 +61,7 @@ class buf_dblwr_t /** mutex protecting the data members below */ mysql_mutex_t mutex; /** condition variable for !batch_running */ - mysql_cond_t cond; + pthread_cond_t cond; /** whether a batch is being written from the doublewrite buffer */ bool batch_running; /** number of expected flush_buffered_writes_completed() calls */ @@ -160,7 +160,7 @@ class buf_dblwr_t { mysql_mutex_lock(&mutex); while (batch_running) - mysql_cond_wait(&cond, &mutex); + my_cond_wait(&cond, &mutex.m_mutex); mysql_mutex_unlock(&mutex); } } diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 44ae69e504255..95167d6a487da 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1084,7 +1084,7 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() if (buf_page_cleaner_is_active) { thread_name = "page cleaner thread"; - mysql_cond_signal(&buf_pool.do_flush_list); + pthread_cond_signal(&buf_pool.do_flush_list); goto wait_suspend_loop; } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index ab4728de9ef8c..7d4e4e6ff978b 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1975,9 +1975,9 @@ void innodb_shutdown() } mysql_mutex_lock(&buf_pool.flush_list_mutex); while (buf_page_cleaner_is_active) { - mysql_cond_signal(&buf_pool.do_flush_list); - mysql_cond_wait(&buf_pool.done_flush_list, - &buf_pool.flush_list_mutex); + pthread_cond_signal(&buf_pool.do_flush_list); + my_cond_wait(&buf_pool.done_flush_list, + &buf_pool.flush_list_mutex.m_mutex); } mysql_mutex_unlock(&buf_pool.flush_list_mutex); break; diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 4fc871040143f..5faba898ba984 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -268,8 +268,9 @@ static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, "The buffer that is allocated when sorting the index when doing a " - "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", NULL, NULL, - SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX, 1); + "REPAIR or when creating indexes with CREATE INDEX or ALTER TABLE.", + NULL, NULL, + SORT_BUFFER_INIT, MIN_SORT_BUFFER, SIZE_T_MAX/2, 1); static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG, "Specifies how Aria index statistics collection code should treat " diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index a0c755b3a7d1b..c63b531934678 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -2359,6 +2359,8 @@ static int initialize_variables_for_repair(HA_CHECK *param, MARIA_SHARE *org_share) { MARIA_SHARE *share= info->s; + size_t tmp; + uint threads; /* We have to clear these variables first, as the cleanup-in-case-of-error @@ -2419,6 +2421,7 @@ static int initialize_variables_for_repair(HA_CHECK *param, /* calculate max_records */ sort_info->filelength= my_seek(info->dfile.file, 0L, MY_SEEK_END, MYF(0)); + param->max_progress= sort_info->filelength; if ((param->testflag & T_CREATE_MISSING_KEYS) || sort_info->org_data_file_type == COMPRESSED_RECORD) @@ -2431,6 +2434,19 @@ static int initialize_variables_for_repair(HA_CHECK *param, sort_info->max_records= (ha_rows) (sort_info->filelength / rec_length); } + /* We don't need a bigger sort buffer than file_length * 8 */ + threads= (param->testflag & T_REP_PARALLEL) ? (uint) share->base.keys : 1; + tmp= (size_t) MY_MIN(sort_info->filelength, + (my_off_t) (SIZE_T_MAX/10/threads)); + tmp= MY_MAX(tmp * 8 * threads, (size_t) 65536); /* Some margin */ + set_if_smaller(param->sort_buffer_length, tmp); + /* Protect against too big sort buffer length */ +#if SIZEOF_SIZE_T >= 8 + set_if_smaller(param->sort_buffer_length, 16LL*1024LL*1024LL*1024LL); +#else + set_if_smaller(param->sort_buffer_length, 1L*1024L*1024L*1024L); +#endif + /* Set up transaction handler so that we can see all rows */ if (param->max_trid == 0) {