From 54324e542f322ffa8d0f70af7ec5c4323a4687c3 Mon Sep 17 00:00:00 2001 From: Daniel Bartholomew Date: Wed, 10 May 2023 08:11:15 -0400 Subject: [PATCH 1/9] bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2abc1c05da39a..91b253ca00cfd 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=4 -MYSQL_VERSION_PATCH=29 +MYSQL_VERSION_PATCH=30 SERVER_MATURITY=stable From aa713f5ae20513b0c4d9a74ea3ba3ea3bbdcd719 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 9 May 2023 21:20:10 -0700 Subject: [PATCH 2/9] MDEV-31224 Crash with EXPLAIN EXTENDED for multi-table update of system table EXPLAIN EXTENDED should always print the field item used in the left part of an equality expression from the SET clause of an update statement as a reference to table column. Approved by Oleksandr Byelkin --- mysql-test/main/explain_non_select.result | 19 ++++++++++++++++++ mysql-test/main/explain_non_select.test | 20 +++++++++++++++++++ .../main/myisam_explain_non_select_all.result | 4 ++-- sql/sql_select.cc | 2 +- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/mysql-test/main/explain_non_select.result b/mysql-test/main/explain_non_select.result index d60f10f85c887..1200a3f9400ca 100644 --- a/mysql-test/main/explain_non_select.result +++ b/mysql-test/main/explain_non_select.result @@ -277,3 +277,22 @@ EXECUTE stmt; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 ALL NULL NULL NULL NULL 2 drop table t1,t2; +# +# MDEV-31224: EXPLAIN EXTENDED for multi-table update of system table +# +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2); +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (3); +EXPLAIN EXTENDED UPDATE t1, t2 SET b = 4 WHERE a IN (6,2); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t2 system NULL NULL NULL NULL 1 100.00 +1 SIMPLE t1 ALL NULL NULL NULL NULL 2 100.00 Using where +Warnings: +Note 1003 update `test`.`t1` set `test`.`t2`.`b` = 4 where `test`.`t1`.`a` in (6,2) +UPDATE t1, t2 SET b = 4 WHERE a IN (6,2); +SELECT * from t2; +b +4 +DROP TABLE t1, t2; +# End of 10.4 tests diff --git a/mysql-test/main/explain_non_select.test b/mysql-test/main/explain_non_select.test index d9ff0fb724503..f87a5d9ec8d67 100644 --- a/mysql-test/main/explain_non_select.test +++ b/mysql-test/main/explain_non_select.test @@ -250,3 +250,23 @@ PREPARE stmt FROM 'EXPLAIN INSERT INTO t1 SELECT * FROM t2'; EXECUTE stmt; drop table t1,t2; +--echo # +--echo # MDEV-31224: EXPLAIN EXTENDED for multi-table update of system table +--echo # + +CREATE TABLE t1 (a INT); +INSERT INTO t1 VALUES (1),(2); + +CREATE TABLE t2 (b INT) ENGINE=MyISAM; +INSERT INTO t2 VALUES (3); + +let $q= +UPDATE t1, t2 SET b = 4 WHERE a IN (6,2); + +eval EXPLAIN EXTENDED $q; +eval $q; +SELECT * from t2; + +DROP TABLE t1, t2; + +--echo # End of 10.4 tests diff --git a/mysql-test/main/myisam_explain_non_select_all.result b/mysql-test/main/myisam_explain_non_select_all.result index 3ca9629d02773..b515cb4fd839a 100644 --- a/mysql-test/main/myisam_explain_non_select_all.result +++ b/mysql-test/main/myisam_explain_non_select_all.result @@ -2689,7 +2689,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t2 system NULL NULL NULL NULL 0 0.00 Const row not found 1 SIMPLE t1 ALL NULL NULL NULL NULL 2 100.00 Warnings: -Note 1003 update `test`.`t1` set NULL = 10 +Note 1003 update `test`.`t1` set `test`.`t2`.`c2` = 10 # Status of EXPLAIN EXTENDED query Variable_name Value Handler_read_key 7 @@ -2734,7 +2734,7 @@ id select_type table type possible_keys key key_len ref rows filtered Extra 1 SIMPLE t2 system NULL NULL NULL NULL 0 0.00 Const row not found 1 SIMPLE t1 ALL NULL NULL NULL NULL 2 100.00 Using where Warnings: -Note 1003 update `test`.`t1` set NULL = 10 where `test`.`t1`.`c3` = 10 +Note 1003 update `test`.`t1` set `test`.`t2`.`c2` = 10 where `test`.`t1`.`c3` = 10 # Status of EXPLAIN EXTENDED query Variable_name Value Handler_read_key 7 diff --git a/sql/sql_select.cc b/sql/sql_select.cc index bfe17bb43e1c5..cb60b3442d89b 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -28138,7 +28138,7 @@ void st_select_lex::print_set_clause(THD *thd, String *str, else str->append(','); - item->print(str, query_type); + item->print(str, (enum_query_type) (query_type | QT_NO_DATA_EXPANSION)); str->append(STRING_WITH_LEN(" = ")); val->print(str, query_type); } From 8f3bf593d24de9cd4744e71c86de80cd502786c7 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 11 May 2023 23:34:41 -0700 Subject: [PATCH 3/9] MDEV-31240 Crash with condition pushable into derived and containing outer reference This bug could affect queries containing a subquery over splittable derived tables and having an outer references in its WHERE clause. If such subquery contained an equality condition whose left part was a reference to a column of the derived table and the right part referred only to outer columns then the server crashed in the function st_join_table::choose_best_splitting() The crashing code was added in the commit ce7ffe61d836fe9f0cfc1087f058bc40d66e5cfb that made the code of the function sensitive to presence of the flag OUTER_REF_TABLE_BIT in the KEYUSE_EXT::needed_in_prefix fields. The field needed_in_prefix of the KEYUSE_EXT structure should not contain table maps with OUTER_REF_TABLE_BIT or RAND_TABLE_BIT. Note that this fix is quite conservative: for affected queries it just returns the query plans that were used before the above mentioned commit. In fact the equalities causing crashes should be pushed into derived tables without any usage of split optimization. Approved by Sergei Petrunia --- mysql-test/main/derived_cond_pushdown.result | 94 ++++++++++++++++++++ mysql-test/main/derived_cond_pushdown.test | 41 +++++++++ sql/opt_split.cc | 3 +- 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/derived_cond_pushdown.result b/mysql-test/main/derived_cond_pushdown.result index 4b202ea7a1262..a4fc7a7447d92 100644 --- a/mysql-test/main/derived_cond_pushdown.result +++ b/mysql-test/main/derived_cond_pushdown.result @@ -18358,4 +18358,98 @@ a deallocate prepare stmt; drop view v1; drop table t1; +# +# MDEV-31240: condition pushed into splittable derived has reference to +# outer column and does not refer to any column of embedding +# select +# +create table t1 (a int); +insert into t1 select seq from seq_1_to_1000; +create table t2 (a int, b int, key (a)); +insert into t2 select mod(seq,100), rand(13) * mod(seq,500) from seq_1_to_1000; +create table t3 (a int); +insert into t3 values (3), (1); +analyze table t1, t2, t3 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status Table is already up to date +test.t3 analyze status Engine-independent statistics collected +test.t3 analyze status OK +explain select +a, +( select concat(t3.a,'=',dt.s) +from +(select a, sum(b) as s from t2 group by a) as dt, +t3 +where dt.a=t1.a and t3.a < 3 +) +from t1 limit 5; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 1000 +2 DEPENDENT SUBQUERY t3 ALL NULL NULL NULL NULL 2 Using where +2 DEPENDENT SUBQUERY ref key0 key0 5 test.t1.a 2 +3 LATERAL DERIVED t2 ref a a 5 test.t1.a 10 +select +a, +( select concat(t3.a,'=',dt.s) +from +(select a, sum(b) as s from t2 group by a) as dt, +t3 +where dt.a=t1.a and t3.a < 3 +) +from t1 limit 5; +a ( select concat(t3.a,'=',dt.s) +from +(select a, sum(b) as s from t2 group by a) as dt, +t3 +where dt.a=t1.a and t3.a < 3 +) +1 1=804 +2 1=1056 +3 1=846 +4 1=947 +5 1=973 +truncate table t2; +insert into t2 select mod(seq,10), rand(15) * mod(seq,500) from seq_1_to_1000; +analyze table t2 persistent for all; +Table Op Msg_type Msg_text +test.t2 analyze status Engine-independent statistics collected +test.t2 analyze status Table is already up to date +explain select +a, +( select concat(t3.a,'=',dt.s) +from +(select a, sum(b) as s from t2 group by a) as dt, +t3 +where dt.a=t1.a and t3.a < 3 +) +from t1 limit 5; +id select_type table type possible_keys key key_len ref rows Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 1000 +2 DEPENDENT SUBQUERY t3 ALL NULL NULL NULL NULL 2 Using where +2 DEPENDENT SUBQUERY ref key0 key0 5 test.t1.a 100 +3 DERIVED t2 ALL a NULL NULL NULL 1000 Using temporary; Using filesort +select +a, +( select concat(t3.a,'=',dt.s) +from +(select a, sum(b) as s from t2 group by a) as dt, +t3 +where dt.a=t1.a and t3.a < 3 +) +from t1 limit 5; +a ( select concat(t3.a,'=',dt.s) +from +(select a, sum(b) as s from t2 group by a) as dt, +t3 +where dt.a=t1.a and t3.a < 3 +) +1 1=11858 +2 1=11380 +3 1=11588 +4 1=11373 +5 1=11612 +drop table t1,t2,t3; # End of 10.4 tests diff --git a/mysql-test/main/derived_cond_pushdown.test b/mysql-test/main/derived_cond_pushdown.test index b4e131dbe796c..7a4d9b4ad7d4b 100644 --- a/mysql-test/main/derived_cond_pushdown.test +++ b/mysql-test/main/derived_cond_pushdown.test @@ -3972,4 +3972,45 @@ deallocate prepare stmt; drop view v1; drop table t1; +--echo # +--echo # MDEV-31240: condition pushed into splittable derived has reference to +--echo # outer column and does not refer to any column of embedding +--echo # select +--echo # + +create table t1 (a int); +insert into t1 select seq from seq_1_to_1000; + +create table t2 (a int, b int, key (a)); +insert into t2 select mod(seq,100), rand(13) * mod(seq,500) from seq_1_to_1000; + +create table t3 (a int); +insert into t3 values (3), (1); + +analyze table t1, t2, t3 persistent for all; + +let $q= +select + a, + ( select concat(t3.a,'=',dt.s) + from + (select a, sum(b) as s from t2 group by a) as dt, + t3 + where dt.a=t1.a and t3.a < 3 + ) +from t1 limit 5; + +eval explain $q; +eval $q; + +truncate table t2; +insert into t2 select mod(seq,10), rand(15) * mod(seq,500) from seq_1_to_1000; + +analyze table t2 persistent for all; + +eval explain $q; +eval $q; + +drop table t1,t2,t3; + --echo # End of 10.4 tests diff --git a/sql/opt_split.cc b/sql/opt_split.cc index bb3aec9ee8d23..6d816552baf41 100644 --- a/sql/opt_split.cc +++ b/sql/opt_split.cc @@ -664,7 +664,8 @@ add_ext_keyuse_for_splitting(Dynamic_array *ext_keyuses, keyuse_ext.cond_guard= added_key_field->cond_guard; keyuse_ext.sj_pred_no= added_key_field->sj_pred_no; keyuse_ext.validity_ref= 0; - keyuse_ext.needed_in_prefix= added_key_field->val->used_tables(); + keyuse_ext.needed_in_prefix= added_key_field->val->used_tables() & + ~(OUTER_REF_TABLE_BIT | RAND_TABLE_BIT); keyuse_ext.validity_var= false; return ext_keyuses->push(keyuse_ext); } From 0fd54c98922a49a0a5ea5df6a4ca5320aaa82103 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 3 Jun 2023 10:15:18 +0200 Subject: [PATCH 4/9] Revert "MDEV-30473 : Do not allow GET_LOCK() / RELEASE_LOCK() in cluster" This reverts commit 844ddb1109410e0ab1d4444549932f1dddb7b845. This fixes MDEV-30967, MDEV-31325, MDEV-31388 --- .../suite/galera/r/galera_locks_funcs.result | 19 ------------------- .../suite/galera/t/galera_locks_funcs.test | 14 -------------- sql/item_create.cc | 14 -------------- 3 files changed, 47 deletions(-) delete mode 100644 mysql-test/suite/galera/r/galera_locks_funcs.result delete mode 100644 mysql-test/suite/galera/t/galera_locks_funcs.test diff --git a/mysql-test/suite/galera/r/galera_locks_funcs.result b/mysql-test/suite/galera/r/galera_locks_funcs.result deleted file mode 100644 index 378067aa46163..0000000000000 --- a/mysql-test/suite/galera/r/galera_locks_funcs.result +++ /dev/null @@ -1,19 +0,0 @@ -connection node_2; -connection node_1; -CREATE TABLE t (c DOUBLE,c2 INT,PRIMARY KEY(c)) ENGINE=InnoDB; -INSERT INTO t values (1,1); -SELECT GET_LOCK('a',1); -ERROR 42000: This version of MariaDB doesn't yet support 'GET_LOCK in cluster (WSREP_ON=ON)' -SHOW WARNINGS; -Level Code Message -Error 1235 This version of MariaDB doesn't yet support 'GET_LOCK in cluster (WSREP_ON=ON)' -SELECT * FROM t; -c c2 -1 1 -SELECT RELEASE_LOCK('a'); -ERROR 42000: This version of MariaDB doesn't yet support 'RELEASE_LOCK in cluster (WSREP_ON=ON)' -SHOW WARNINGS; -Level Code Message -Error 1235 This version of MariaDB doesn't yet support 'RELEASE_LOCK in cluster (WSREP_ON=ON)' -COMMIT; -DROP TABLE t; diff --git a/mysql-test/suite/galera/t/galera_locks_funcs.test b/mysql-test/suite/galera/t/galera_locks_funcs.test deleted file mode 100644 index 68737f2daabb1..0000000000000 --- a/mysql-test/suite/galera/t/galera_locks_funcs.test +++ /dev/null @@ -1,14 +0,0 @@ ---source include/galera_cluster.inc - -CREATE TABLE t (c DOUBLE,c2 INT,PRIMARY KEY(c)) ENGINE=InnoDB; -INSERT INTO t values (1,1); ---error ER_NOT_SUPPORTED_YET -SELECT GET_LOCK('a',1); -SHOW WARNINGS; -SELECT * FROM t; ---error ER_NOT_SUPPORTED_YET -SELECT RELEASE_LOCK('a'); -SHOW WARNINGS; -COMMIT; -DROP TABLE t; - diff --git a/sql/item_create.cc b/sql/item_create.cc index 616ba3a641ad9..29d9563d1d141 100644 --- a/sql/item_create.cc +++ b/sql/item_create.cc @@ -4933,13 +4933,6 @@ Create_func_get_lock Create_func_get_lock::s_singleton; Item* Create_func_get_lock::create_2_arg(THD *thd, Item *arg1, Item *arg2) { -#ifdef WITH_WSREP - if (WSREP_ON && WSREP(thd)) - { - my_error(ER_NOT_SUPPORTED_YET, MYF(0), "GET_LOCK in cluster (WSREP_ON=ON)"); - return NULL; - } -#endif /* WITH_WSREP */ thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); thd->lex->uncacheable(UNCACHEABLE_SIDEEFFECT); return new (thd->mem_root) Item_func_get_lock(thd, arg1, arg2); @@ -6542,13 +6535,6 @@ Create_func_release_lock Create_func_release_lock::s_singleton; Item* Create_func_release_lock::create_1_arg(THD *thd, Item *arg1) { -#ifdef WITH_WSREP - if (WSREP_ON && WSREP(thd)) - { - my_error(ER_NOT_SUPPORTED_YET, MYF(0), "RELEASE_LOCK in cluster (WSREP_ON=ON)"); - return NULL; - } -#endif /* WITH_WSREP */ thd->lex->set_stmt_unsafe(LEX::BINLOG_STMT_UNSAFE_SYSTEM_FUNCTION); thd->lex->uncacheable(UNCACHEABLE_SIDEEFFECT); return new (thd->mem_root) Item_func_release_lock(thd, arg1); From eb472f77e3f6696bbdbe48d15e71a2c444edc5e1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 3 Jun 2023 10:35:59 +0200 Subject: [PATCH 5/9] Revert "MDEV-30473 : Do not allow GET_LOCK() / RELEASE_LOCK() in cluster" This reverts commit b05218e08f45a17f46d1b73cbb9dcb2969dc04cd. --- mysql-test/suite/galera/r/MDEV-24143.result | 23 ++++++++++++ .../galera/r/galera_bf_abort_get_lock.result | 18 ++++++++++ mysql-test/suite/galera/t/MDEV-24143.test | 20 +++++++++++ .../galera/t/galera_bf_abort_get_lock.test | 36 +++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 mysql-test/suite/galera/r/MDEV-24143.result create mode 100644 mysql-test/suite/galera/r/galera_bf_abort_get_lock.result create mode 100644 mysql-test/suite/galera/t/MDEV-24143.test create mode 100644 mysql-test/suite/galera/t/galera_bf_abort_get_lock.test diff --git a/mysql-test/suite/galera/r/MDEV-24143.result b/mysql-test/suite/galera/r/MDEV-24143.result new file mode 100644 index 0000000000000..860d8a35834c0 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-24143.result @@ -0,0 +1,23 @@ +connection node_2; +connection node_1; +CREATE TABLE t1 (c1 BIGINT NOT NULL PRIMARY KEY, c2 BINARY (10), c3 DATETIME); +SELECT get_lock ('test2', 0); +get_lock ('test2', 0) +1 +DROP TABLE t1; +CREATE TABLE t1 (c1 SMALLINT NOT NULL AUTO_INCREMENT PRIMARY KEY); +INSERT INTO t1 VALUES (1); +SET SESSION wsrep_trx_fragment_size=10; +SET SESSION autocommit=0; +SELECT * FROM t1 WHERE c1 <=0 ORDER BY c1 DESC; +c1 +INSERT INTO t1 VALUES (4),(3),(1),(2); +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +CREATE TABLE t1 (pk INT PRIMARY KEY, b INT) ENGINE=SEQUENCE; +ERROR 42S01: Table 't1' already exists +ALTER TABLE t1 DROP COLUMN c2; +ERROR 42000: Can't DROP COLUMN `c2`; check that it exists +SELECT get_lock ('test', 1.5); +get_lock ('test', 1.5) +1 +DROP TABLE t1; diff --git a/mysql-test/suite/galera/r/galera_bf_abort_get_lock.result b/mysql-test/suite/galera/r/galera_bf_abort_get_lock.result new file mode 100644 index 0000000000000..0ef2a1a72c648 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_bf_abort_get_lock.result @@ -0,0 +1,18 @@ +connection node_2; +connection node_1; +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; +connection node_2a; +SELECT GET_LOCK("foo", 1000); +GET_LOCK("foo", 1000) +1 +connection node_2; +SET AUTOCOMMIT=OFF; +INSERT INTO t1 VALUES (1); +SELECT GET_LOCK("foo", 1000);; +connection node_1; +INSERT INTO t1 VALUES (1); +connection node_2; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +wsrep_local_aborts_increment +1 +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MDEV-24143.test b/mysql-test/suite/galera/t/MDEV-24143.test new file mode 100644 index 0000000000000..e58f147cb7c3c --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-24143.test @@ -0,0 +1,20 @@ +--source include/galera_cluster.inc +--source include/have_sequence.inc + +CREATE TABLE t1 (c1 BIGINT NOT NULL PRIMARY KEY, c2 BINARY (10), c3 DATETIME); +SELECT get_lock ('test2', 0); +DROP TABLE t1; +CREATE TABLE t1 (c1 SMALLINT NOT NULL AUTO_INCREMENT PRIMARY KEY); +INSERT INTO t1 VALUES (1); +SET SESSION wsrep_trx_fragment_size=10; +SET SESSION autocommit=0; +SELECT * FROM t1 WHERE c1 <=0 ORDER BY c1 DESC; +--error ER_LOCK_DEADLOCK +INSERT INTO t1 VALUES (4),(3),(1),(2); +--error ER_TABLE_EXISTS_ERROR +CREATE TABLE t1 (pk INT PRIMARY KEY, b INT) ENGINE=SEQUENCE; +--error ER_CANT_DROP_FIELD_OR_KEY +ALTER TABLE t1 DROP COLUMN c2; +SELECT get_lock ('test', 1.5); +DROP TABLE t1; + diff --git a/mysql-test/suite/galera/t/galera_bf_abort_get_lock.test b/mysql-test/suite/galera/t/galera_bf_abort_get_lock.test new file mode 100644 index 0000000000000..72fc1c5b583bb --- /dev/null +++ b/mysql-test/suite/galera/t/galera_bf_abort_get_lock.test @@ -0,0 +1,36 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +# +# Test a local transaction being aborted by a slave one while it is running a GET_LOCK() +# + +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB; + +--let $galera_connection_name = node_2a +--let $galera_server_number = 2 +--source include/galera_connect.inc +--connection node_2a +SELECT GET_LOCK("foo", 1000); + +--connection node_2 +SET AUTOCOMMIT=OFF; +--let $wsrep_local_bf_aborts_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'` +INSERT INTO t1 VALUES (1); +--send SELECT GET_LOCK("foo", 1000); + +--connection node_1 +INSERT INTO t1 VALUES (1); + +--connection node_2 +--error ER_LOCK_DEADLOCK +--reap + +--let $wsrep_local_bf_aborts_after = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'` + +# Check that wsrep_local_bf_aborts has been incremented by exactly 1 +--disable_query_log +--eval SELECT $wsrep_local_bf_aborts_after - $wsrep_local_bf_aborts_before = 1 AS wsrep_local_aborts_increment; +--enable_query_log + +DROP TABLE t1; From 318012a80a5b4365b4942edf2723742503a9fd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 19 May 2023 12:19:26 +0300 Subject: [PATCH 6/9] MDEV-31234 InnoDB does not free UNDO after the fix of MDEV-30671 trx_purge_truncate_history(): Only call trx_purge_truncate_rseg_history() if the rollback segment is safe to process. This will avoid leaking undo log pages that are not yet ready to be processed. This fixes a regression that was introduced in commit 0de3be8cfdfc26f5c236eaefe12d03c7b4af22c8 (MDEV-30671). trx_sys_t::any_active_transactions(): Separately count XA PREPARE transactions. srv_purge_should_exit(): Terminate slow shutdown if the history size does not change and XA PREPARE transactions exist in the system. This will avoid a hang of the test innodb.recovery_shutdown. Tested by: Matthias Leich --- storage/innobase/include/trx0sys.h | 2 +- storage/innobase/srv/srv0srv.cc | 41 ++++++++++++++++++------------ storage/innobase/trx/trx0purge.cc | 10 +++----- storage/innobase/trx/trx0sys.cc | 26 ++++++++++++++----- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index e033a3e1fe4c0..016ac0b136329 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -1055,7 +1055,7 @@ class trx_sys_t void close(); /** @return total number of active (non-prepared) transactions */ - ulint any_active_transactions(); + size_t any_active_transactions(size_t *prepared= nullptr); /** diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 57aa4bef9fe95..50569f810ea30 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1697,7 +1697,7 @@ void srv_master_callback(void*) } /** @return whether purge should exit due to shutdown */ -static bool srv_purge_should_exit() +static bool srv_purge_should_exit(size_t old_history_size) { ut_ad(srv_shutdown_state <= SRV_SHUTDOWN_CLEANUP); @@ -1708,7 +1708,12 @@ static bool srv_purge_should_exit() return true; /* Slow shutdown was requested. */ - if (const size_t history_size= trx_sys.rseg_history_len) + size_t prepared, active= trx_sys.any_active_transactions(&prepared); + const size_t history_size= trx_sys.rseg_history_len; + + if (!history_size); + else if (!active && history_size == old_history_size && prepared); + else { static time_t progress_time; time_t now= time(NULL); @@ -1725,7 +1730,7 @@ static bool srv_purge_should_exit() return false; } - return !trx_sys.any_active_transactions(); + return !active; } /*********************************************************************//** @@ -1845,7 +1850,7 @@ static size_t srv_do_purge(ulint* n_total_purged) *n_total_purged += n_pages_purged; } while (n_pages_purged > 0 && !purge_sys.paused() - && !srv_purge_should_exit()); + && !srv_purge_should_exit(rseg_history_len)); return(rseg_history_len); } @@ -1960,7 +1965,7 @@ static void purge_coordinator_callback_low() } } while ((purge_sys.enabled() && !purge_sys.paused()) || - !srv_purge_should_exit()); + !srv_purge_should_exit(trx_sys.rseg_history_len)); } static void purge_coordinator_callback(void*) @@ -2031,15 +2036,19 @@ ulint srv_get_task_queue_length() /** Shut down the purge threads. */ void srv_purge_shutdown() { - if (purge_sys.enabled()) { - if (!srv_fast_shutdown && !opt_bootstrap) - srv_update_purge_thread_count(innodb_purge_threads_MAX); - while(!srv_purge_should_exit()) { - ut_a(!purge_sys.paused()); - srv_wake_purge_thread_if_not_active(); - purge_coordinator_task.wait(); - } - purge_sys.coordinator_shutdown(); - srv_shutdown_purge_tasks(); - } + if (purge_sys.enabled()) + { + if (!srv_fast_shutdown && !opt_bootstrap) + srv_update_purge_thread_count(innodb_purge_threads_MAX); + size_t history_size= trx_sys.rseg_history_len; + while (!srv_purge_should_exit(history_size)) + { + history_size= trx_sys.rseg_history_len; + ut_a(!purge_sys.paused()); + srv_wake_purge_thread_if_not_active(); + purge_coordinator_task.wait(); + } + purge_sys.coordinator_shutdown(); + srv_shutdown_purge_tasks(); + } } diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index b22a85f4646ca..97979a3fefe90 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -448,12 +448,7 @@ trx_purge_truncate_rseg_history( prev_hdr_addr.boffset = static_cast(prev_hdr_addr.boffset - TRX_UNDO_HISTORY_NODE); - if (!rseg.trx_ref_count - && rseg.needs_purge <= (purge_sys.head.trx_no - ? purge_sys.head.trx_no - : purge_sys.tail.trx_no) - && mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE - + block->frame) + if (mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE + block->frame) == TRX_UNDO_TO_PURGE && !mach_read_from_2(block->frame + hdr_addr.boffset + TRX_UNDO_NEXT_LOG)) { @@ -544,7 +539,8 @@ static void trx_purge_truncate_history() ut_ad(rseg->id == i); ut_ad(rseg->is_persistent()); mutex_enter(&rseg->mutex); - trx_purge_truncate_rseg_history(*rseg, head); + if (!rseg->trx_ref_count && rseg->needs_purge <= head.trx_no) + trx_purge_truncate_rseg_history(*rseg, head); mutex_exit(&rseg->mutex); } } diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index bcde969eb41cf..ab3de55db64aa 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -325,15 +325,29 @@ trx_sys_t::close() } /** @return total number of active (non-prepared) transactions */ -ulint trx_sys_t::any_active_transactions() +size_t trx_sys_t::any_active_transactions(size_t *prepared) { - uint32_t total_trx= 0; - - trx_sys.trx_list.for_each([&total_trx](const trx_t &trx) { - if (trx.state == TRX_STATE_COMMITTED_IN_MEMORY || - (trx.state == TRX_STATE_ACTIVE && trx.id)) + size_t total_trx= 0, prepared_trx= 0; + + trx_sys.trx_list.for_each([&](const trx_t &trx) { + switch (trx.state) { + case TRX_STATE_NOT_STARTED: + break; + case TRX_STATE_ACTIVE: + if (!trx.id) + break; + /* fall through */ + case TRX_STATE_COMMITTED_IN_MEMORY: total_trx++; + break; + case TRX_STATE_PREPARED: + case TRX_STATE_PREPARED_RECOVERED: + prepared_trx++; + } }); + if (prepared) + *prepared= prepared_trx; + return total_trx; } From 48d6a5f61bde7065cc04a9d225514535780b34a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 24 May 2023 08:25:26 +0300 Subject: [PATCH 7/9] MDEV-31234 fixup: Free some UNDO pages earlier trx_purge_truncate_rseg_history(): Add a parameter to specify if the entire rollback segment is safe to be freed. If not, we may still be able to invoke trx_undo_truncate_start() and free some pages. --- storage/innobase/trx/trx0purge.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 97979a3fefe90..c5ec42ecfce8e 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -399,12 +399,14 @@ void trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) /** Remove unnecessary history data from a rollback segment. @param[in,out] rseg rollback segment -@param[in] limit truncate anything before this */ +@param[in] limit truncate anything before this +@param[in] all whether everything can be truncated */ static void trx_purge_truncate_rseg_history( trx_rseg_t& rseg, - const purge_sys_t::iterator& limit) + const purge_sys_t::iterator& limit, + bool all) { fil_addr_t hdr_addr; fil_addr_t prev_hdr_addr; @@ -443,6 +445,10 @@ trx_purge_truncate_rseg_history( goto func_exit; } + if (!all) { + goto func_exit; + } + prev_hdr_addr = flst_get_prev_addr(block->frame + hdr_addr.boffset + TRX_UNDO_HISTORY_NODE); prev_hdr_addr.boffset = static_cast(prev_hdr_addr.boffset @@ -539,8 +545,9 @@ static void trx_purge_truncate_history() ut_ad(rseg->id == i); ut_ad(rseg->is_persistent()); mutex_enter(&rseg->mutex); - if (!rseg->trx_ref_count && rseg->needs_purge <= head.trx_no) - trx_purge_truncate_rseg_history(*rseg, head); + trx_purge_truncate_rseg_history(*rseg, head, + !rseg->trx_ref_count && + rseg->needs_purge <= head.trx_no); mutex_exit(&rseg->mutex); } } From 3b4b512d8e42bb3d33cf78789754cd10ff310646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 23 May 2023 12:20:27 +0300 Subject: [PATCH 8/9] MDEV-31234 fixup: Allow innodb_undo_log_truncate=ON after upgrade trx_purge_truncate_history(): Relax a condition that would prevent undo log truncation if the undo log tablespaces were "contaminated" by the bug that commit e0084b9d315f10e3ceb578b65e144d751b208bf1 fixed. That is, trx_purge_truncate_rseg_history() would have invoked flst_remove() on TRX_RSEG_HISTORY but not reduced TRX_RSEG_HISTORY_SIZE. To avoid any regression with normal operation, we implement this fixup during slow shutdown only. The condition on the history list being empty is necessary: without it, in the test innodb.undo_truncate_recover there may be much fewer than the expected 90,000 calls to row_purge() before the truncation. That is, we would truncate the undo tablespace before actually having processed all undo log records in it. To truncate such "contaminated" or "bloated" undo log tablespaces (when using innodb_undo_tablespaces=2 or more) you can execute the following SQL: BEGIN;INSERT mysql.innodb_table_stats VALUES('','',DEFAULT,0,0,0);ROLLBACK; SET GLOBAL innodb_undo_log_truncate=ON, innodb_fast_shutdown=0; SHUTDOWN; The first line creates a dummy InnoDB transaction, to ensure that there will be some history to be purged during shutdown and that the undo tablespaces will be truncated. --- storage/innobase/trx/trx0purge.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index c5ec42ecfce8e..2f21a4de1e6f3 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -626,7 +626,8 @@ static void trx_purge_truncate_history() ut_ad(rseg->curr_size > cached); - if (rseg->curr_size > cached + 1) + if (rseg->curr_size > cached + 1 && + (srv_fast_shutdown || srv_undo_sources || trx_sys.rseg_history_len)) goto not_free; mutex_exit(&rseg->mutex); From 928012a27ae2d1549f1b3e6975b9d22652bbea3a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 5 Jun 2023 16:40:08 +0300 Subject: [PATCH 9/9] MDEV-31403: Server crashes in st_join_table::choose_best_splitting The code in choose_best_splitting() assumed that the join prefix is in join->positions[]. This is not necessarily the case. This function might be called when the join prefix is in join->best_positions[], too. Follow the approach from best_access_path(), which calls this function: pass the current join prefix as an argument, "const POSITION *join_positions" and use that. --- mysql-test/main/derived_split_innodb.result | 15 +++++++++++++++ mysql-test/main/derived_split_innodb.test | 21 +++++++++++++++++++++ sql/opt_split.cc | 3 ++- sql/sql_select.cc | 1 + sql/sql_select.h | 1 + 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/derived_split_innodb.result b/mysql-test/main/derived_split_innodb.result index 736e6a2c02034..bea2271eaf358 100644 --- a/mysql-test/main/derived_split_innodb.result +++ b/mysql-test/main/derived_split_innodb.result @@ -810,4 +810,19 @@ SELECT t1.* FROM t1 JOIN (SELECT id, COUNT(*) FROM t2 GROUP BY id) sq ON sq.id= a set optimizer_switch= @tmp1, join_cache_level= @tmp2; DROP TABLE t1, t2; +# +# MDEV-31403: Server crashes in st_join_table::choose_best_splitting (still) +# +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES +(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15); +CREATE TABLE t2 (b INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (100),(200); +CREATE TABLE t3 (c INT, d INT, KEY(c)) ENGINE=InnoDB; +INSERT INTO t3 VALUES (1,1),(2,2); +CREATE VIEW v AS SELECT c, d FROM t3 GROUP BY c, d; +SELECT * FROM t1 JOIN t2 WHERE (t1.a, t2.b) IN (SELECT * FROM v); +a b +DROP VIEW v; +DROP TABLE t1, t2, t3; # End of 10.4 tests diff --git a/mysql-test/main/derived_split_innodb.test b/mysql-test/main/derived_split_innodb.test index b8dd5ad20e1a9..3c51f54fe0221 100644 --- a/mysql-test/main/derived_split_innodb.test +++ b/mysql-test/main/derived_split_innodb.test @@ -462,4 +462,25 @@ set optimizer_switch= @tmp1, join_cache_level= @tmp2; # Cleanup DROP TABLE t1, t2; +--echo # +--echo # MDEV-31403: Server crashes in st_join_table::choose_best_splitting (still) +--echo # +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +INSERT INTO t1 VALUES + (1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15); + +CREATE TABLE t2 (b INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (100),(200); + +CREATE TABLE t3 (c INT, d INT, KEY(c)) ENGINE=InnoDB; +INSERT INTO t3 VALUES (1,1),(2,2); + +CREATE VIEW v AS SELECT c, d FROM t3 GROUP BY c, d; + +SELECT * FROM t1 JOIN t2 WHERE (t1.a, t2.b) IN (SELECT * FROM v); + +# Cleanup +DROP VIEW v; +DROP TABLE t1, t2, t3; + --echo # End of 10.4 tests diff --git a/sql/opt_split.cc b/sql/opt_split.cc index 6d816552baf41..89ffe35aba1e8 100644 --- a/sql/opt_split.cc +++ b/sql/opt_split.cc @@ -948,6 +948,7 @@ void reset_validity_vars_for_keyuses(KEYUSE_EXT *key_keyuse_ext_start, SplM_plan_info * JOIN_TAB::choose_best_splitting(uint idx, table_map remaining_tables, + const POSITION *join_positions, table_map *spl_pd_boundary) { SplM_opt_info *spl_opt_info= table->spl_opt_info; @@ -1045,7 +1046,7 @@ SplM_plan_info * JOIN_TAB::choose_best_splitting(uint idx, else { table_map last_found= this->table->map; - for (POSITION *pos= &this->join->positions[idx - 1]; ; pos--) + for (const POSITION *pos= &join_positions[idx - 1]; ; pos--) { if (pos->table->table->map & excluded_tables) continue; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index cb60b3442d89b..2834c771ca3e6 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -7452,6 +7452,7 @@ best_access_path(JOIN *join, if (s->table->is_splittable()) spl_plan= s->choose_best_splitting(idx, remaining_tables, + join_positions, &spl_pd_boundary); Json_writer_array trace_paths(thd, "considered_access_paths"); diff --git a/sql/sql_select.h b/sql/sql_select.h index f4b266292c9f3..37c784fc9b92c 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -694,6 +694,7 @@ typedef struct st_join_table { void add_keyuses_for_splitting(); SplM_plan_info *choose_best_splitting(uint idx, table_map remaining_tables, + const POSITION *join_positions, table_map *spl_pd_boundary); bool fix_splitting(SplM_plan_info *spl_plan, table_map excluded_tables, bool is_const_table);