From 54ed3939f7fc8008aac72999b6c5fc5dae9c0467 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 22 Dec 2023 23:09:27 -0800 Subject: [PATCH 01/33] MDEV-31657 Crash on query using CTE with the same name as a base table If a query contained a CTE whose name coincided with the name of one of the base tables used in the specification of the CTE and the query had at least two references to this CTE in the specifications of other CTEs then processing of the query led to unlimited recursion that ultimately caused a crash of the server. Any secondary non-recursive reference to a CTE requires creation of a copy of the CTE specification. All the references to CTEs in this copy must be resolved. If the specification contains a reference to a base table whose name coincides with the name of then CTE then it should be ensured that this reference in no way can be resolved against the name of the CTE. --- mysql-test/main/cte_nonrecursive.result | 45 ++++++++++++++++++++++++ mysql-test/main/cte_nonrecursive.test | 46 +++++++++++++++++++++++++ sql/sql_acl.cc | 5 --- sql/sql_cte.cc | 34 ++++++++++++------ sql/sql_cte.h | 6 ++-- sql/sql_lex.h | 6 ++-- sql/sql_view.cc | 3 +- 7 files changed, 124 insertions(+), 21 deletions(-) diff --git a/mysql-test/main/cte_nonrecursive.result b/mysql-test/main/cte_nonrecursive.result index e86c101686cbb..f499b7a532122 100644 --- a/mysql-test/main/cte_nonrecursive.result +++ b/mysql-test/main/cte_nonrecursive.result @@ -2624,4 +2624,49 @@ a 1 1 DROP TABLE t1; +# +# MDEV-31657: CTE with the same name as base table used twice +# in another CTE +# +create table t (a int); +insert into t values (3), (7), (1); +with +t as (select * from t), +cte as (select t1.a as t1a, t2.a as t2a from t as t1, t as t2 where t1.a=t2.a) +select * from cte; +t1a t2a +3 3 +7 7 +1 1 +create table s (a int); +insert into s values (1), (4), (7); +with +t as (select * from t), +s as (select a-1 as a from s), +cte as (select t.a as ta, s.a as sa from t, s where t.a=s.a +union +select t.a+1, s.a+1 from t, s where t.a=s.a+1) +select * from cte; +ta sa +3 3 +2 1 +8 7 +with +t as (select * from t), +cte as (select t.a as ta, s.a as sa from t, s where t.a=s.a +union +select t.a+1, s.a+1 from t, s where t.a=s.a), +s as (select a+10 as a from s) +select * from cte; +ta sa +1 1 +7 7 +2 2 +8 8 +drop table t,s; +with +t as (select * from t), +cte as (select t1.a as t1a, t2.a as t2a from t as t1, t as t2 where t1.a=t2.a) +select * from cte; +ERROR 42S02: Table 'test.t' doesn't exist # End of 10.4 tests diff --git a/mysql-test/main/cte_nonrecursive.test b/mysql-test/main/cte_nonrecursive.test index a666ed3a25f14..0d342edf97d58 100644 --- a/mysql-test/main/cte_nonrecursive.test +++ b/mysql-test/main/cte_nonrecursive.test @@ -1967,4 +1967,50 @@ SELECT * FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-31657: CTE with the same name as base table used twice +--echo # in another CTE +--echo # + +create table t (a int); +insert into t values (3), (7), (1); + +let $q1= +with +t as (select * from t), +cte as (select t1.a as t1a, t2.a as t2a from t as t1, t as t2 where t1.a=t2.a) +select * from cte; + +eval $q1; + +create table s (a int); +insert into s values (1), (4), (7); + +let $q2= +with +t as (select * from t), +s as (select a-1 as a from s), +cte as (select t.a as ta, s.a as sa from t, s where t.a=s.a + union + select t.a+1, s.a+1 from t, s where t.a=s.a+1) +select * from cte; + +eval $q2; + +let $q3= +with +t as (select * from t), +cte as (select t.a as ta, s.a as sa from t, s where t.a=s.a + union + select t.a+1, s.a+1 from t, s where t.a=s.a), +s as (select a+10 as a from s) +select * from cte; + +eval $q3; + +drop table t,s; + +--ERROR ER_NO_SUCH_TABLE +eval $q1; + --echo # End of 10.4 tests diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 82f8101e929fc..6a6a6b3984dba 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -8137,11 +8137,6 @@ bool check_grant(THD *thd, ulong want_access, TABLE_LIST *tables, INSERT_ACL : SELECT_ACL); } - if (tl->with || !tl->db.str || - (tl->select_lex && - (tl->with= tl->select_lex->find_table_def_in_with_clauses(tl)))) - continue; - const ACL_internal_table_access *access= get_cached_table_access(&t_ref->grant.m_internal, t_ref->get_db_name(), diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index 0e422b30216c3..ce735ba26d14b 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -105,6 +105,7 @@ bool LEX::check_dependencies_in_with_clauses() @param tables Points to the beginning of the sub-chain @param tables_last Points to the address with the sub-chain barrier + @param excl_spec Ignore the definition with this spec @details The method resolves tables references to CTE from the chain of @@ -146,7 +147,8 @@ bool LEX::check_dependencies_in_with_clauses() */ bool LEX::resolve_references_to_cte(TABLE_LIST *tables, - TABLE_LIST **tables_last) + TABLE_LIST **tables_last, + st_select_lex_unit *excl_spec) { With_element *with_elem= 0; @@ -155,7 +157,8 @@ bool LEX::resolve_references_to_cte(TABLE_LIST *tables, if (tbl->derived) continue; if (!tbl->db.str && !tbl->with) - tbl->with= tbl->select_lex->find_table_def_in_with_clauses(tbl); + tbl->with= tbl->select_lex->find_table_def_in_with_clauses(tbl, + excl_spec); if (!tbl->with) // no CTE matches table reference tbl { if (only_cte_resolution) @@ -243,7 +246,7 @@ LEX::check_cte_dependencies_and_resolve_references() return true; if (!with_cte_resolution) return false; - if (resolve_references_to_cte(query_tables, query_tables_last)) + if (resolve_references_to_cte(query_tables, query_tables_last, NULL)) return true; return false; } @@ -387,6 +390,7 @@ bool With_element::check_dependencies_in_spec() @param table The reference to the table that is looked for @param barrier The barrier with element for the search + @param excl_spec Ignore the definition with this spec @details The function looks through the elements of this with clause trying to find @@ -400,12 +404,15 @@ bool With_element::check_dependencies_in_spec() */ With_element *With_clause::find_table_def(TABLE_LIST *table, - With_element *barrier) + With_element *barrier, + st_select_lex_unit *excl_spec) { for (With_element *with_elem= with_list.first; with_elem != barrier; with_elem= with_elem->next) { + if (excl_spec && with_elem->spec == excl_spec) + continue; if (my_strcasecmp(system_charset_info, with_elem->get_name_str(), table->table_name.str) == 0 && !table->is_fqtn) @@ -465,7 +472,7 @@ With_element *find_table_def_in_with_clauses(TABLE_LIST *tbl, top_unit->with_element && top_unit->with_element->get_owner() == with_clause) barrier= top_unit->with_element; - found= with_clause->find_table_def(tbl, barrier); + found= with_clause->find_table_def(tbl, barrier, NULL); if (found) break; } @@ -520,10 +527,11 @@ void With_element::check_dependencies_in_select(st_select_lex *sl, { With_clause *with_clause= sl->master_unit()->with_clause; if (with_clause) - tbl->with= with_clause->find_table_def(tbl, NULL); + tbl->with= with_clause->find_table_def(tbl, NULL, NULL); if (!tbl->with) tbl->with= owner->find_table_def(tbl, - owner->with_recursive ? NULL : this); + owner->with_recursive ? NULL : this, + NULL); } if (!tbl->with) tbl->with= find_table_def_in_with_clauses(tbl, ctxt); @@ -1098,7 +1106,8 @@ st_select_lex_unit *With_element::clone_parsed_spec(LEX *old_lex, */ lex->only_cte_resolution= old_lex->only_cte_resolution; if (lex->resolve_references_to_cte(lex->query_tables, - lex->query_tables_last)) + lex->query_tables_last, + spec)) { res= NULL; goto err; @@ -1264,6 +1273,7 @@ bool With_element::is_anchor(st_select_lex *sel) Search for the definition of the given table referred in this select node @param table reference to the table whose definition is searched for + @param excl_spec ignore the definition with this spec @details The method looks for the definition of the table whose reference is occurred @@ -1276,7 +1286,8 @@ bool With_element::is_anchor(st_select_lex *sel) NULL - otherwise */ -With_element *st_select_lex::find_table_def_in_with_clauses(TABLE_LIST *table) +With_element *st_select_lex::find_table_def_in_with_clauses(TABLE_LIST *table, + st_select_lex_unit *excl_spec) { With_element *found= NULL; With_clause *containing_with_clause= NULL; @@ -1293,7 +1304,7 @@ With_element *st_select_lex::find_table_def_in_with_clauses(TABLE_LIST *table) With_clause *attached_with_clause= sl->get_with_clause(); if (attached_with_clause && attached_with_clause != containing_with_clause && - (found= attached_with_clause->find_table_def(table, NULL))) + (found= attached_with_clause->find_table_def(table, NULL, excl_spec))) break; master_unit= sl->master_unit(); outer_sl= master_unit->outer_select(); @@ -1303,7 +1314,8 @@ With_element *st_select_lex::find_table_def_in_with_clauses(TABLE_LIST *table) containing_with_clause= with_elem->get_owner(); With_element *barrier= containing_with_clause->with_recursive ? NULL : with_elem; - if ((found= containing_with_clause->find_table_def(table, barrier))) + if ((found= containing_with_clause->find_table_def(table, barrier, + excl_spec))) break; if (outer_sl && !outer_sl->get_with_element()) break; diff --git a/sql/sql_cte.h b/sql/sql_cte.h index 4bc3ed69582cc..4d56672e3cc58 100644 --- a/sql/sql_cte.h +++ b/sql/sql_cte.h @@ -322,7 +322,8 @@ class With_element : public Sql_alloc friend bool LEX::resolve_references_to_cte(TABLE_LIST *tables, - TABLE_LIST **tables_last); + TABLE_LIST **tables_last, + st_select_lex_unit *excl_spec); }; const uint max_number_of_elements_in_with_clause= sizeof(table_map)*8; @@ -422,7 +423,8 @@ class With_clause : public Sql_alloc void move_anchors_ahead(); - With_element *find_table_def(TABLE_LIST *table, With_element *barrier); + With_element *find_table_def(TABLE_LIST *table, With_element *barrier, + st_select_lex_unit *excl_spec); With_element *find_table_def_in_with_clauses(TABLE_LIST *table); diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 0f2e096fdc675..2305616d5b366 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -1531,7 +1531,8 @@ class st_select_lex: public st_select_lex_node master_unit()->cloned_from->with_element : master_unit()->with_element; } - With_element *find_table_def_in_with_clauses(TABLE_LIST *table); + With_element *find_table_def_in_with_clauses(TABLE_LIST *table, + st_select_lex_unit * excl_spec); bool check_unrestricted_recursive(bool only_standard_compliant); bool check_subqueries_with_recursive_references(); void collect_grouping_fields_for_derived(THD *thd, ORDER *grouping_list); @@ -4708,7 +4709,8 @@ struct LEX: public Query_tables_list bool check_dependencies_in_with_clauses(); bool check_cte_dependencies_and_resolve_references(); bool resolve_references_to_cte(TABLE_LIST *tables, - TABLE_LIST **tables_last); + TABLE_LIST **tables_last, + st_select_lex_unit *excl_spec); }; diff --git a/sql/sql_view.cc b/sql/sql_view.cc index 9f9a97169824a..0dca5f7c226bb 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -295,7 +295,8 @@ bool create_view_precheck(THD *thd, TABLE_LIST *tables, TABLE_LIST *view, for (tbl= sl->get_table_list(); tbl; tbl= tbl->next_local) { if (!tbl->with && tbl->select_lex) - tbl->with= tbl->select_lex->find_table_def_in_with_clauses(tbl); + tbl->with= tbl->select_lex->find_table_def_in_with_clauses(tbl, + NULL); /* Ensure that we have some privileges on this table, more strict check will be done on column level after preparation, From 613d0194979849fb5b3dd752f13b14672a2409e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 4 Jan 2024 12:50:05 +0200 Subject: [PATCH 02/33] MDEV-33160 show_status_array() calls various functions via incompatible pointer In commit b4ff64568c88ab3ce559e7bd39853d9cbf86704a the signature of mysql_show_var_func was changed, but not all functions of that type were adjusted. When the server is configured with `cmake -DWITH_ASAN=ON` and compiled with clang, runtime errors would be flagged for invoking functions through an incompatible function pointer. Reviewed by: Michael 'Monty' Widenius --- sql/mysqld.cc | 144 +++++++++++++++++------------------- sql/sql_acl.cc | 8 +- sql/table_cache.cc | 4 +- sql/table_cache.h | 4 +- sql/threadpool.h | 3 - storage/sphinx/ha_sphinx.cc | 19 +++-- storage/sphinx/ha_sphinx.h | 6 -- storage/spider/spd_param.cc | 18 +++-- 8 files changed, 102 insertions(+), 104 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a1a8a862a8e6e..5713045056dd8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -7180,8 +7180,8 @@ struct my_option my_long_options[]= MYSQL_TO_BE_IMPLEMENTED_OPTION("validate-user-plugins") // NO_EMBEDDED_ACCESS_CHECKS }; -static int show_queries(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_queries(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { var->type= SHOW_LONGLONG; var->value= &thd->query_id; @@ -7189,16 +7189,16 @@ static int show_queries(THD *thd, SHOW_VAR *var, char *buff, } -static int show_net_compression(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_net_compression(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { var->type= SHOW_MY_BOOL; var->value= &thd->net.compress; return 0; } -static int show_starttime(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_starttime(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7207,8 +7207,8 @@ static int show_starttime(THD *thd, SHOW_VAR *var, char *buff, } #ifdef ENABLED_PROFILING -static int show_flushstatustime(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_flushstatustime(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7218,32 +7218,28 @@ static int show_flushstatustime(THD *thd, SHOW_VAR *var, char *buff, #endif #ifdef HAVE_REPLICATION -static int show_rpl_status(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_rpl_status(THD *, SHOW_VAR *var, void *, system_status_var *, + enum_var_type) { var->type= SHOW_CHAR; var->value= const_cast(rpl_status_type[(int)rpl_status]); return 0; } -static int show_slave_running(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_slave_running(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - Master_info *mi= NULL; - bool UNINIT_VAR(tmp); - - var->type= SHOW_MY_BOOL; - var->value= buff; - - if ((mi= get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE))) + if (Master_info *mi= + get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE)) { - tmp= (my_bool) (mi->slave_running == MYSQL_SLAVE_RUN_READING && - mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN); + *((my_bool*) buff)= + (mi->slave_running == MYSQL_SLAVE_RUN_READING && + mi->rli.slave_running != MYSQL_SLAVE_NOT_RUN); mi->release(); + var->type= SHOW_MY_BOOL; + var->value= buff; } - if (mi) - *((my_bool *)buff)= tmp; else var->type= SHOW_UNDEF; return 0; @@ -7253,7 +7249,8 @@ static int show_slave_running(THD *thd, SHOW_VAR *var, char *buff, /* How many masters this slave is connected to */ -static int show_slaves_running(THD *thd, SHOW_VAR *var, char *buff) +static int show_slaves_running(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONGLONG; var->value= buff; @@ -7264,19 +7261,17 @@ static int show_slaves_running(THD *thd, SHOW_VAR *var, char *buff) } -static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - Master_info *mi; - - var->type= SHOW_LONGLONG; - var->value= buff; - - if ((mi= get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE))) + if (Master_info *mi= + get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE)) { *((longlong *)buff)= mi->received_heartbeats; mi->release(); + var->type= SHOW_LONGLONG; + var->value= buff; } else var->type= SHOW_UNDEF; @@ -7284,19 +7279,17 @@ static int show_slave_received_heartbeats(THD *thd, SHOW_VAR *var, char *buff, } -static int show_heartbeat_period(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_heartbeat_period(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { - Master_info *mi; - - var->type= SHOW_CHAR; - var->value= buff; - - if ((mi= get_master_info(&thd->variables.default_master_connection, - Sql_condition::WARN_LEVEL_NOTE))) + if (Master_info *mi= + get_master_info(&thd->variables.default_master_connection, + Sql_condition::WARN_LEVEL_NOTE)) { - sprintf(buff, "%.3f", mi->heartbeat_period); + sprintf(static_cast(buff), "%.3f", mi->heartbeat_period); mi->release(); + var->type= SHOW_CHAR; + var->value= buff; } else var->type= SHOW_UNDEF; @@ -7306,8 +7299,8 @@ static int show_heartbeat_period(THD *thd, SHOW_VAR *var, char *buff, #endif /* HAVE_REPLICATION */ -static int show_open_tables(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_open_tables(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7315,8 +7308,8 @@ static int show_open_tables(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_prepared_stmt_count(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_prepared_stmt_count(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7326,8 +7319,8 @@ static int show_prepared_stmt_count(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_table_definitions(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_table_definitions(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7336,8 +7329,8 @@ static int show_table_definitions(THD *thd, SHOW_VAR *var, char *buff, } -static int show_flush_commands(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_flush_commands(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONGLONG; var->value= buff; @@ -7356,8 +7349,8 @@ static int show_flush_commands(THD *thd, SHOW_VAR *var, char *buff, inside an Event. */ -static int show_ssl_get_version(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_ssl_get_version(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { var->type= SHOW_CHAR; if( thd->vio_ok() && thd->net.vio->ssl_arg ) @@ -7367,8 +7360,8 @@ static int show_ssl_get_version(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_ssl_get_default_timeout(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_ssl_get_default_timeout(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7379,8 +7372,8 @@ static int show_ssl_get_default_timeout(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_ssl_get_verify_mode(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_ssl_get_verify_mode(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7395,8 +7388,8 @@ static int show_ssl_get_verify_mode(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_ssl_get_verify_depth(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_ssl_get_verify_depth(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_LONG; var->value= buff; @@ -7408,8 +7401,8 @@ static int show_ssl_get_verify_depth(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_ssl_get_cipher(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_ssl_get_cipher(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_CHAR; if( thd->vio_ok() && thd->net.vio->ssl_arg ) @@ -7419,9 +7412,10 @@ static int show_ssl_get_cipher(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_ssl_get_cipher_list(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_ssl_get_cipher_list(THD *thd, SHOW_VAR *var, void *buf, + system_status_var *, enum_var_type) { + char *buff= static_cast(buf); var->type= SHOW_CHAR; var->value= buff; if (thd->vio_ok() && thd->net.vio->ssl_arg) @@ -7506,8 +7500,8 @@ my_asn1_time_to_string(const ASN1_TIME *time, char *buf, size_t len) */ static int -show_ssl_get_server_not_before(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +show_ssl_get_server_not_before(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_CHAR; if(thd->vio_ok() && thd->net.vio->ssl_arg) @@ -7516,7 +7510,7 @@ show_ssl_get_server_not_before(THD *thd, SHOW_VAR *var, char *buff, X509 *cert= SSL_get_certificate(ssl); const ASN1_TIME *not_before= X509_get0_notBefore(cert); - var->value= my_asn1_time_to_string(not_before, buff, + var->value= my_asn1_time_to_string(not_before, static_cast(buff), SHOW_VAR_FUNC_BUFF_SIZE); if (!var->value) return 1; @@ -7540,8 +7534,8 @@ show_ssl_get_server_not_before(THD *thd, SHOW_VAR *var, char *buff, */ static int -show_ssl_get_server_not_after(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +show_ssl_get_server_not_after(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_CHAR; if(thd->vio_ok() && thd->net.vio->ssl_arg) @@ -7550,7 +7544,7 @@ show_ssl_get_server_not_after(THD *thd, SHOW_VAR *var, char *buff, X509 *cert= SSL_get_certificate(ssl); const ASN1_TIME *not_after= X509_get0_notAfter(cert); - var->value= my_asn1_time_to_string(not_after, buff, + var->value= my_asn1_time_to_string(not_after, static_cast(buff), SHOW_VAR_FUNC_BUFF_SIZE); if (!var->value) return 1; @@ -7604,7 +7598,7 @@ static int show_default_keycache(THD *thd, SHOW_VAR *var, void *buff, } -static int show_memory_used(THD *thd, SHOW_VAR *var, char *buff, +static int show_memory_used(THD *thd, SHOW_VAR *var, void *buff, struct system_status_var *status_var, enum enum_var_type scope) { @@ -7660,8 +7654,8 @@ static int debug_status_func(THD *thd, SHOW_VAR *var, void *buff, #endif #ifdef HAVE_POOL_OF_THREADS -int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +int show_threadpool_idle_threads(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_INT; var->value= buff; @@ -7670,8 +7664,8 @@ int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff, } -static int show_threadpool_threads(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_threadpool_threads(THD *, SHOW_VAR *var, void *buff, + system_status_var *, enum_var_type) { var->type= SHOW_INT; var->value= buff; diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index 6a6a6b3984dba..1949a0c02b6a8 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -11869,8 +11869,8 @@ static my_bool count_column_grants(void *grant_table, This must be performed under the mutex in order to make sure the iteration does not fail. */ -static int show_column_grants(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_column_grants(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum enum_var_type scope) { var->type= SHOW_ULONG; var->value= buff; @@ -11886,8 +11886,8 @@ static int show_column_grants(THD *thd, SHOW_VAR *var, char *buff, return 0; } -static int show_database_grants(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +static int show_database_grants(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum enum_var_type scope) { var->type= SHOW_UINT; var->value= buff; diff --git a/sql/table_cache.cc b/sql/table_cache.cc index 9b28e7b7b0ff5..ca7c25ae552f7 100644 --- a/sql/table_cache.cc +++ b/sql/table_cache.cc @@ -1345,8 +1345,8 @@ int tdc_iterate(THD *thd, my_hash_walk_action action, void *argument, } -int show_tc_active_instances(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope) +int show_tc_active_instances(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum enum_var_type scope) { var->type= SHOW_UINT; var->value= buff; diff --git a/sql/table_cache.h b/sql/table_cache.h index 3be7b5fe413b6..ee738eb8ba6da 100644 --- a/sql/table_cache.h +++ b/sql/table_cache.h @@ -97,8 +97,8 @@ extern int tdc_iterate(THD *thd, my_hash_walk_action action, void *argument, bool no_dups= false); extern uint tc_records(void); -int show_tc_active_instances(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope); +int show_tc_active_instances(THD *thd, SHOW_VAR *var, void *buff, + system_status_var *, enum enum_var_type scope); extern void tc_purge(bool mark_flushed= false); extern void tc_add_table(THD *thd, TABLE *table); extern void tc_release_table(TABLE *table); diff --git a/sql/threadpool.h b/sql/threadpool.h index ea6c93b4a65ea..e432132716316 100644 --- a/sql/threadpool.h +++ b/sql/threadpool.h @@ -64,9 +64,6 @@ extern int tp_get_thread_count(); /* Activate threadpool scheduler */ extern void tp_scheduler(void); -extern int show_threadpool_idle_threads(THD *thd, SHOW_VAR *var, char *buff, - enum enum_var_type scope); - enum TP_PRIORITY { TP_PRIORITY_HIGH, TP_PRIORITY_LOW, diff --git a/storage/sphinx/ha_sphinx.cc b/storage/sphinx/ha_sphinx.cc index f2bc24c47d4ce..4549e8dccb6de 100644 --- a/storage/sphinx/ha_sphinx.cc +++ b/storage/sphinx/ha_sphinx.cc @@ -3548,7 +3548,8 @@ CSphSEStats * sphinx_get_stats ( THD * thd, SHOW_VAR * out ) return 0; } -int sphinx_showfunc_total ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_total ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3559,7 +3560,8 @@ int sphinx_showfunc_total ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_total_found ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_total_found ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3570,7 +3572,8 @@ int sphinx_showfunc_total_found ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_time ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_time ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3581,7 +3584,8 @@ int sphinx_showfunc_time ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_word_count ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_word_count ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); if ( pStats ) @@ -3592,9 +3596,11 @@ int sphinx_showfunc_word_count ( THD * thd, SHOW_VAR * out, char * ) return 0; } -int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, char * sBuffer ) +static int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, void * buf, + system_status_var *, enum_var_type ) { #if MYSQL_VERSION_ID>50100 + char *sBuffer = static_cast(buf); if ( sphinx_hton_ptr ) { CSphTLS * pTls = (CSphTLS *) *thd_ha_data ( thd, sphinx_hton_ptr ); @@ -3649,7 +3655,8 @@ int sphinx_showfunc_words ( THD * thd, SHOW_VAR * out, char * sBuffer ) return 0; } -int sphinx_showfunc_error ( THD * thd, SHOW_VAR * out, char * ) +static int sphinx_showfunc_error ( THD * thd, SHOW_VAR * out, void *, + system_status_var *, enum_var_type ) { CSphSEStats * pStats = sphinx_get_stats ( thd, out ); out->type = SHOW_CHAR; diff --git a/storage/sphinx/ha_sphinx.h b/storage/sphinx/ha_sphinx.h index decd88bad5a84..ddc1567328f92 100644 --- a/storage/sphinx/ha_sphinx.h +++ b/storage/sphinx/ha_sphinx.h @@ -164,12 +164,6 @@ class ha_sphinx : public handler bool sphinx_show_status ( THD * thd ); #endif -int sphinx_showfunc_total_found ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_total ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_time ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_word_count ( THD *, SHOW_VAR *, char * ); -int sphinx_showfunc_words ( THD *, SHOW_VAR *, char * ); - // // $Id: ha_sphinx.h 4818 2014-09-24 08:53:38Z tomat $ // diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc index 5eaf95180488e..5c86af68c0a85 100644 --- a/storage/spider/spd_param.cc +++ b/storage/spider/spd_param.cc @@ -115,7 +115,8 @@ extern volatile ulonglong spider_mon_table_cache_version_req; } #ifdef HANDLER_HAS_DIRECT_UPDATE_ROWS -static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) +static int spider_direct_update(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { int error_num = 0; SPIDER_TRX *trx; @@ -126,7 +127,8 @@ static int spider_direct_update(THD *thd, SHOW_VAR *var, char *buff) DBUG_RETURN(error_num); } -static int spider_direct_delete(THD *thd, SHOW_VAR *var, char *buff) +static int spider_direct_delete(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { int error_num = 0; SPIDER_TRX *trx; @@ -138,7 +140,8 @@ static int spider_direct_delete(THD *thd, SHOW_VAR *var, char *buff) } #endif -static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, char *buff) +static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { int error_num = 0; SPIDER_TRX *trx; @@ -149,7 +152,8 @@ static int spider_direct_order_limit(THD *thd, SHOW_VAR *var, char *buff) DBUG_RETURN(error_num); } -static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, char *buff) +static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { int error_num = 0; SPIDER_TRX *trx; @@ -160,7 +164,8 @@ static int spider_direct_aggregate(THD *thd, SHOW_VAR *var, char *buff) DBUG_RETURN(error_num); } -static int spider_parallel_search(THD *thd, SHOW_VAR *var, char *buff) +static int spider_parallel_search(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { int error_num = 0; SPIDER_TRX *trx; @@ -172,7 +177,8 @@ static int spider_parallel_search(THD *thd, SHOW_VAR *var, char *buff) } #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET) -static int spider_hs_result_free(THD *thd, SHOW_VAR *var, char *buff) +static int spider_hs_result_free(THD *thd, SHOW_VAR *var, void *, + system_status_var *, enum_var_type) { int error_num = 0; SPIDER_TRX *trx; From ac0ce4451958a5c95c9b2ede242586c600836f81 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 3 Jan 2024 12:04:50 +0100 Subject: [PATCH 03/33] ./mtr --skip-not-found should skip combinations too With the result like encryption.innochecksum 'debug' [ skipped ] combination not found instead of *** ERROR: Could not run encryption.innochecksum with 'debug' combination(s) --- mysql-test/lib/mtr_cases.pm | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm index a4473a07e814d..666ac74791e0f 100644 --- a/mysql-test/lib/mtr_cases.pm +++ b/mysql-test/lib/mtr_cases.pm @@ -892,6 +892,12 @@ sub collect_one_test_case { } my @no_combs = grep { $test_combs{$_} == 1 } keys %test_combs; if (@no_combs) { + if ($::opt_skip_not_found) { + push @{$tinfo->{combinations}}, @no_combs; + $tinfo->{'skip'}= 1; + $tinfo->{'comment'}= "combination not found"; + return $tinfo; + } mtr_error("Could not run $name with '".( join(',', sort @no_combs))."' combination(s)"); } From 8172d07785245ad129dfaa9fecca4da206fd3e07 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 3 Jan 2024 18:36:36 +0100 Subject: [PATCH 04/33] MDEV-33090 plugin/auth_pam/testing/pam_mariadb_mtr.c doesn't compile on Solaris Fix by Rainer Orth --- plugin/auth_pam/testing/pam_mariadb_mtr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/auth_pam/testing/pam_mariadb_mtr.c b/plugin/auth_pam/testing/pam_mariadb_mtr.c index d4c79f3133065..dc2bc150dc9c7 100644 --- a/plugin/auth_pam/testing/pam_mariadb_mtr.c +++ b/plugin/auth_pam/testing/pam_mariadb_mtr.c @@ -10,8 +10,8 @@ #include #include -#include #include +#include #define N 3 From f7573e7a837af75768b518936f99c4df634e1b79 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 3 Jan 2024 18:55:16 +0100 Subject: [PATCH 05/33] MDEV-33093 plugin/disks/information_schema_disks.cc doesn't compile on Solaris Fix by Rainer Orth --- plugin/disks/information_schema_disks.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugin/disks/information_schema_disks.cc b/plugin/disks/information_schema_disks.cc index 01c6c0e173feb..1be8fce37fa92 100644 --- a/plugin/disks/information_schema_disks.cc +++ b/plugin/disks/information_schema_disks.cc @@ -19,10 +19,14 @@ #include #if defined(HAVE_GETMNTENT) #include +#elif defined(HAVE_SYS_MNTENT) +#include #elif !defined(HAVE_GETMNTINFO_TAKES_statvfs) /* getmntinfo (the not NetBSD variants) */ #include +#if defined(HAVE_SYS_UCRED) #include +#endif #include #endif #if defined(HAVE_GETMNTENT_IN_SYS_MNTAB) From ca276a0f3fcb45ff0abc011e334c700e0c5d4315 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Fri, 5 Jan 2024 09:35:57 +1100 Subject: [PATCH 06/33] MDEV-33169 Reset sequence used fields after check in alter sequence The bitmap is temporarily flipped to ~0 for the sake of checking all fields. It needs to be restored because it will be reused in second and subsequent ps execution. --- mysql-test/suite/sql_sequence/alter.result | 49 ++++++++++++++++++++++ mysql-test/suite/sql_sequence/alter.test | 35 ++++++++++++++++ sql/sql_sequence.cc | 2 + 3 files changed, 86 insertions(+) diff --git a/mysql-test/suite/sql_sequence/alter.result b/mysql-test/suite/sql_sequence/alter.result index 68d42e5278479..d4e930ebb9244 100644 --- a/mysql-test/suite/sql_sequence/alter.result +++ b/mysql-test/suite/sql_sequence/alter.result @@ -252,3 +252,52 @@ SELECT NEXTVAL(s); NEXTVAL(s) 1 DROP SEQUENCE s; +# +# MDEV-33169 Alter sequence 2nd ps fails while alter sequence 2nd time (no ps) succeeds +# +create sequence s; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=MyISAM +alter sequence s maxvalue 123; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 123 increment by 1 cache 1000 nocycle ENGINE=MyISAM +alter sequence s maxvalue 123; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 123 increment by 1 cache 1000 nocycle ENGINE=MyISAM +drop sequence s; +create sequence s; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=MyISAM +prepare stmt from 'alter sequence s maxvalue 123'; +execute stmt; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 123 increment by 1 cache 1000 nocycle ENGINE=MyISAM +execute stmt; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 123 increment by 1 cache 1000 nocycle ENGINE=MyISAM +deallocate prepare stmt; +drop sequence s; +create sequence s; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 9223372036854775806 increment by 1 cache 1000 nocycle ENGINE=MyISAM +create procedure p() alter sequence s maxvalue 123; +call p; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 123 increment by 1 cache 1000 nocycle ENGINE=MyISAM +call p; +show create sequence s; +Table Create Table +s CREATE SEQUENCE `s` start with 1 minvalue 1 maxvalue 123 increment by 1 cache 1000 nocycle ENGINE=MyISAM +drop procedure p; +drop sequence s; +# +# End of 10.4 tests +# diff --git a/mysql-test/suite/sql_sequence/alter.test b/mysql-test/suite/sql_sequence/alter.test index a771c9bba2f78..015aba22af6e4 100644 --- a/mysql-test/suite/sql_sequence/alter.test +++ b/mysql-test/suite/sql_sequence/alter.test @@ -167,3 +167,38 @@ ALTER TABLE s ORDER BY cache_size; SELECT NEXTVAL(s); DROP SEQUENCE s; --enable_ps2_protocol + +--echo # +--echo # MDEV-33169 Alter sequence 2nd ps fails while alter sequence 2nd time (no ps) succeeds +--echo # +create sequence s; +show create sequence s; +alter sequence s maxvalue 123; +show create sequence s; +alter sequence s maxvalue 123; +show create sequence s; +drop sequence s; + +create sequence s; +show create sequence s; +prepare stmt from 'alter sequence s maxvalue 123'; +execute stmt; +show create sequence s; +execute stmt; +show create sequence s; +deallocate prepare stmt; +drop sequence s; + +create sequence s; +show create sequence s; +create procedure p() alter sequence s maxvalue 123; +call p; +show create sequence s; +call p; +show create sequence s; +drop procedure p; +drop sequence s; + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/sql/sql_sequence.cc b/sql/sql_sequence.cc index ab77eabfa2832..405b2d5b003ba 100644 --- a/sql/sql_sequence.cc +++ b/sql/sql_sequence.cc @@ -924,6 +924,7 @@ bool Sql_cmd_alter_sequence::execute(THD *thd) TABLE_LIST *first_table= lex->query_tables; TABLE *table; sequence_definition *new_seq= lex->create_info.seq_create_info; + uint saved_used_fields= new_seq->used_fields; SEQUENCE *seq; No_such_table_error_handler no_such_table_handler; DBUG_ENTER("Sql_cmd_alter_sequence::execute"); @@ -1043,5 +1044,6 @@ bool Sql_cmd_alter_sequence::execute(THD *thd) my_ok(thd); end: + new_seq->used_fields= saved_used_fields; DBUG_RETURN(error); } From 9322ef03e339ee8fcea25231c73c2f63d56c0d49 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 13 Nov 2023 11:18:16 +0400 Subject: [PATCH 07/33] MDEV-32645 CAST(AS UNSIGNED) fails with --view-protocol Item_float::neg() did not preserve the "presentation" from "this". So CAST(-1e0 AS UNSIGNED) -- cast from double to unsigned changes its meaning to: CAST(-1 AS UNSIGNED) -- cast signed to undigned Fixing Item_float::neg() to construct the new value for Item_float::presentation as follows: - if the old value starts with minus, then the minus is truncated: '-2e0' -> '2e0' - otherwise, minus sign followed by its old value: '1e0' -> '-1e0' --- mysql-test/main/cast.test | 3 --- mysql-test/main/type_float.result | 37 +++++++++++++++++++++++++++++++ mysql-test/main/type_float.test | 26 ++++++++++++++++++++++ sql/item.cc | 20 ++++++++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/mysql-test/main/cast.test b/mysql-test/main/cast.test index ce4b1f6a5749e..5a0f87beb696d 100644 --- a/mysql-test/main/cast.test +++ b/mysql-test/main/cast.test @@ -768,14 +768,11 @@ INSERT INTO t1 VALUES (-1.0); SELECT * FROM t1; DROP TABLE t1; -#enable after MDEV-32645 is fixed ---disable_view_protocol SELECT CAST(-1e0 AS UNSIGNED); CREATE TABLE t1 (a BIGINT UNSIGNED); INSERT INTO t1 VALUES (-1e0); SELECT * FROM t1; DROP TABLE t1; ---enable_view_protocol SELECT CAST(-1e308 AS UNSIGNED); CREATE TABLE t1 (a BIGINT UNSIGNED); diff --git a/mysql-test/main/type_float.result b/mysql-test/main/type_float.result index 500f906642ddd..fa850cf9b347c 100644 --- a/mysql-test/main/type_float.result +++ b/mysql-test/main/type_float.result @@ -1167,5 +1167,42 @@ d 50 fdbl 123.456.789,12345678000000000000000000000000000000 fdec 123.456.789,12345678900000000000000000000000000000 # +# MDEV-32645 CAST(AS UNSIGNED) fails with --view-protocol +# +SELECT +CAST(-1e0 AS UNSIGNED), +CAST(--2e0 AS UNSIGNED), +CAST(---3e0 AS UNSIGNED), +CAST(----4e0 AS UNSIGNED); +CAST(-1e0 AS UNSIGNED) CAST(--2e0 AS UNSIGNED) CAST(---3e0 AS UNSIGNED) CAST(----4e0 AS UNSIGNED) +0 2 0 4 +Warnings: +Note 1916 Got overflow when converting '-1' to UNSIGNED BIGINT. Value truncated +Note 1916 Got overflow when converting '-3' to UNSIGNED BIGINT. Value truncated +EXPLAIN EXTENDED SELECT +CAST(-1e0 AS UNSIGNED), +CAST(--2e0 AS UNSIGNED), +CAST(---3e0 AS UNSIGNED), +CAST(----4e0 AS UNSIGNED); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL NULL No tables used +Warnings: +Note 1003 select cast(-1e0 as unsigned) AS `CAST(-1e0 AS UNSIGNED)`,cast(2e0 as unsigned) AS `CAST(--2e0 AS UNSIGNED)`,cast(-3e0 as unsigned) AS `CAST(---3e0 AS UNSIGNED)`,cast(4e0 as unsigned) AS `CAST(----4e0 AS UNSIGNED)` +CREATE VIEW v1 AS SELECT +CAST(-1e0 AS UNSIGNED), +CAST(--2e0 AS UNSIGNED), +CAST(---3e0 AS UNSIGNED), +CAST(----4e0 AS UNSIGNED); +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select cast(-1e0 as unsigned) AS `CAST(-1e0 AS UNSIGNED)`,cast(2e0 as unsigned) AS `CAST(--2e0 AS UNSIGNED)`,cast(-3e0 as unsigned) AS `CAST(---3e0 AS UNSIGNED)`,cast(4e0 as unsigned) AS `CAST(----4e0 AS UNSIGNED)` latin1 latin1_swedish_ci +SELECT * FROM v1; +CAST(-1e0 AS UNSIGNED) CAST(--2e0 AS UNSIGNED) CAST(---3e0 AS UNSIGNED) CAST(----4e0 AS UNSIGNED) +0 2 0 4 +Warnings: +Note 1916 Got overflow when converting '-1' to UNSIGNED BIGINT. Value truncated +Note 1916 Got overflow when converting '-3' to UNSIGNED BIGINT. Value truncated +DROP VIEW v1; +# # End of 10.4 tests # diff --git a/mysql-test/main/type_float.test b/mysql-test/main/type_float.test index f1041080e268c..f1d74f2f8d714 100644 --- a/mysql-test/main/type_float.test +++ b/mysql-test/main/type_float.test @@ -713,6 +713,32 @@ $$ DELIMITER ;$$ --horizontal_results +--echo # +--echo # MDEV-32645 CAST(AS UNSIGNED) fails with --view-protocol +--echo # + +SELECT + CAST(-1e0 AS UNSIGNED), + CAST(--2e0 AS UNSIGNED), + CAST(---3e0 AS UNSIGNED), + CAST(----4e0 AS UNSIGNED); + +EXPLAIN EXTENDED SELECT + CAST(-1e0 AS UNSIGNED), + CAST(--2e0 AS UNSIGNED), + CAST(---3e0 AS UNSIGNED), + CAST(----4e0 AS UNSIGNED); + +CREATE VIEW v1 AS SELECT + CAST(-1e0 AS UNSIGNED), + CAST(--2e0 AS UNSIGNED), + CAST(---3e0 AS UNSIGNED), + CAST(----4e0 AS UNSIGNED); + +SHOW CREATE VIEW v1; +SELECT * FROM v1; +DROP VIEW v1; + --echo # --echo # End of 10.4 tests --echo # diff --git a/sql/item.cc b/sql/item.cc index 21190b38e1aa8..6d30d63bc1167 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -6896,7 +6896,25 @@ Item *Item_float::neg(THD *thd) else if (value < 0 && max_length) max_length--; value= -value; - presentation= 0; + if (presentation) + { + if (*presentation == '-') + { + // Strip double minus: -(-1) -> '1' instead of '--1' + presentation++; + } + else + { + size_t presentation_length= strlen(presentation); + if (char *tmp= (char*) thd->alloc(presentation_length + 2)) + { + tmp[0]= '-'; + // Copy with the trailing '\0' + memcpy(tmp + 1, presentation, presentation_length + 1); + presentation= tmp; + } + } + } name= null_clex_str; return this; } From c6e1ffd1a07fc451e7211b0d00edbace78137276 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 31 Dec 2023 23:30:48 +0100 Subject: [PATCH 08/33] MDEV-33148 A connection can control RAND() in following connection initialize THD::rand in THD::init() not in THD::THD(), because the former is also called when a THD is reused - in COM_CHANGE_USER and in taking a THD from the cache. Also use current cycle timer for more unpreditability --- sql/sql_class.cc | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index b4893581e1aca..179e2a1a9a573 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -684,7 +684,6 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) wsrep_wfc() #endif /*WITH_WSREP */ { - ulong tmp; bzero(&variables, sizeof(variables)); /* @@ -834,14 +833,6 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) tablespace_op=FALSE; - /* - Initialize the random generator. We call my_rnd() without a lock as - it's not really critical if two threads modifies the structure at the - same time. We ensure that we have an unique number foreach thread - by adding the address of the stack. - */ - tmp= (ulong) (my_rnd(&sql_rand) * 0xffffffff); - my_rnd_init(&rand, tmp + (ulong)((size_t) &rand), tmp + (ulong) ::global_query_id); substitute_null_with_insert_id = FALSE; lock_info.mysql_thd= (void *)this; @@ -1297,6 +1288,17 @@ void THD::init() /* Set to handle counting of aborted connections */ userstat_running= opt_userstat_running; last_global_update_time= current_connect_time= time(NULL); + + /* + Initialize the random generator. We call my_rnd() without a lock as + it's not really critical if two threads modify the structure at the + same time. We ensure that we have a unique number for each thread + by adding the address of this THD. + */ + ulong tmp= (ulong) (my_rnd(&sql_rand) * 0xffffffff); + my_rnd_init(&rand, tmp + (ulong)(intptr) this, + (ulong)(my_timer_cycles() + global_query_id)); + #if defined(ENABLED_DEBUG_SYNC) /* Initialize the Debug Sync Facility. See debug_sync.cc. */ debug_sync_init_thread(this); From 2310f659f528a189bb9c2a8e70a63e7ee7702780 Mon Sep 17 00:00:00 2001 From: Rainer Orth Date: Wed, 10 Jan 2024 10:11:43 +1100 Subject: [PATCH 09/33] MDEV-8941 Compile on Solaris (SPARC) fails with errors in filamvct.cpp There are a large number of uses of `strerror` in the codebase, the local declaration in `storage/connect/tabvct.cpp` is the only one. Given that none is needed elsewhere, I conclude that this instance can simply be removed. --- storage/connect/tabvct.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/storage/connect/tabvct.cpp b/storage/connect/tabvct.cpp index 9cf5f41d1feec..f5710688d3c5c 100644 --- a/storage/connect/tabvct.cpp +++ b/storage/connect/tabvct.cpp @@ -71,11 +71,6 @@ #include "tabvct.h" #include "valblk.h" -#if defined(UNIX) -//add dummy strerror (NGC) -char *strerror(int num); -#endif // UNIX - /***********************************************************************/ /* External function. */ /***********************************************************************/ From eabc74aaef472cb415f4ead502559ca1c27efbc8 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 10 Jan 2024 16:36:39 +1100 Subject: [PATCH 10/33] MDEV-33008 Fix spider table discovery A new column was introduced to the show index output in 10.6 in f691d9865becfd242ba44cc632034433336af1e7 Thus we update the check of the number of columns to be at least 13, rather than exactly 13. Also backport an err number and format from 10.5 for better error messages when the column number is wrong. --- .../spider/bugfix/r/mdev_33008.result | 25 +++++++++++++++++++ .../spider/bugfix/t/mdev_33008.test | 24 ++++++++++++++++++ storage/spider/spd_db_mysql.cc | 9 ++++--- storage/spider/spd_err.h | 2 ++ 4 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result new file mode 100644 index 0000000000000..3bcb4bb038ab8 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_33008.result @@ -0,0 +1,25 @@ +for master_1 +for child2 +for child3 +set spider_same_server_link=on; +CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 ( +`id` int(11) NOT NULL AUTO_INCREMENT, +`code` varchar(10) DEFAULT NULL, +PRIMARY KEY (`id`) +); +create table t1 ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `code` varchar(10) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=SPIDER DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"' +drop table t1, t2; +drop server srv; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test new file mode 100644 index 0000000000000..48d9a4f01ec72 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_33008.test @@ -0,0 +1,24 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +set spider_same_server_link=on; +evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `code` varchar(10) DEFAULT NULL, + PRIMARY KEY (`id`) +); +create table t1 ENGINE=Spider +COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +show create table t1; +drop table t1, t2; + +drop server srv; +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index e689c9d93f04b..d2e9dc1f0ee89 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -1566,10 +1566,13 @@ int spider_db_mbase_result::fetch_index_for_discover_table_structure( } DBUG_RETURN(0); } - if (num_fields() != 13) + if (num_fields() < 13) { - DBUG_PRINT("info",("spider num_fields != 13")); - my_printf_error(ER_SPIDER_UNKNOWN_NUM, ER_SPIDER_UNKNOWN_STR, MYF(0)); + DBUG_PRINT("info",("spider num_fields < 13")); + my_printf_error(ER_SPIDER_CANT_NUM, ER_SPIDER_CANT_STR1, MYF(0), + "fetch index for table structure discovery because of " + "wrong number of columns in SHOW INDEX FROM output: ", + num_fields()); DBUG_RETURN(ER_SPIDER_UNKNOWN_NUM); } bool first = TRUE; diff --git a/storage/spider/spd_err.h b/storage/spider/spd_err.h index 9889fcfa7fb5c..d8f11e18f26ea 100644 --- a/storage/spider/spd_err.h +++ b/storage/spider/spd_err.h @@ -127,6 +127,8 @@ #define ER_SPIDER_SAME_SERVER_LINK_NUM 12720 #define ER_SPIDER_SAME_SERVER_LINK_STR1 "Host:%s and Socket:%s aim self server. Please change spider_same_server_link parameter if this link is required." #define ER_SPIDER_SAME_SERVER_LINK_STR2 "Host:%s and Port:%ld aim self server. Please change spider_same_server_link parameter if this link is required." +#define ER_SPIDER_CANT_NUM 12721 +#define ER_SPIDER_CANT_STR1 "Can't %s%d" #define ER_SPIDER_COND_SKIP_NUM 12801 #define ER_SPIDER_UNKNOWN_NUM 12500 From bc3d416a17c9d35382f2db6387e51619e80c59da Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Wed, 10 Jan 2024 16:37:36 +1100 Subject: [PATCH 11/33] MDEV-29718 Fix spider detection of same data node server When the host is not specified, it defaults to localhost. --- storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result | 4 ++++ .../mysql-test/spider/bugfix/r/mdev_28739_simple.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result | 1 + storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result | 1 + .../spider/bugfix/r/spider_join_with_non_spider.result | 1 + storage/spider/mysql-test/spider/bugfix/r/subquery.result | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test | 5 +++++ .../spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test | 1 + storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test | 1 + .../spider/bugfix/t/spider_join_with_non_spider.test | 1 + storage/spider/mysql-test/spider/bugfix/t/subquery.test | 1 + storage/spider/spd_db_mysql.cc | 4 ++-- 33 files changed, 41 insertions(+), 2 deletions(-) diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result index b0a430e0e212e..326b84a030e4b 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_26151.result @@ -6,6 +6,9 @@ for child2 for child3 set @old_spider_bgs_mode= @@spider_bgs_mode; set session spider_bgs_mode=1; +set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table td (a int, PRIMARY KEY (a)); create table ts (a int, PRIMARY KEY (a)) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv_mdev_26151",TABLE "td", casual_read "3"'; @@ -26,6 +29,7 @@ min(a) drop table td, ts; drop server srv_mdev_26151; set session spider_bgs_mode=@old_spider_bgs_mode; +set global spider_same_server_link=@old_spider_same_server_link; for master_1 for child2 for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result index db232f8a6d373..1c337c3d2dd4c 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28739_simple.result @@ -5,6 +5,7 @@ for master_1 for child2 for child3 set global query_cache_type= on; +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t2 (c int); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result index fae3cc6b6ce76..7e4fd3cd08466 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28856.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # testing monitoring_* diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result index edac80d97616e..e92fb19957567 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_28998.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (1),(2); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result index af4bef1efa987..f58ab605e111c 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29163.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); CREATE TABLE t2 (b INT); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result index 4d9095830d1e5..365c3d6373ad3 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29456.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t1 (c int); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result index 17390346a9934..604515964f3fb 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29963.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t (a INT) ENGINE=Spider; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result index e1dca495047c7..ee53009e3f24d 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30014.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t1 (c int); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result index 58873d2c6e55c..cefa5248d44f7 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30392.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (1),(2); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result index 62b06336ff6b5..f156cf38a15c2 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31338.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t (c BLOB) ENGINE=InnoDB; CREATE TABLE ts (c BLOB) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result index 645fc62862d94..b3a4c7525652c 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31524.result @@ -5,6 +5,7 @@ for master_1 for child2 for child3 SET @old_spider_read_only_mode = @@session.spider_read_only_mode; +set spider_same_server_link=1; CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); set session spider_read_only_mode = default; create table t2 (c int); diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result index 94b76de7df477..5197abd3fb6dc 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31645.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)); CREATE TABLE t2 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)) ENGINE=SPIDER COMMENT='srv "srv", WRAPPER "mysql", TABLE "t1"'; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result index 04d7e88467635..cbc91432dbc3b 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_31996.result @@ -1,6 +1,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); set session spider_delete_all_rows_type=0; diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result index bdc580d421ac4..c3bdef9870d92 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_32986.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t2 (c varchar(16)); diff --git a/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result b/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result index b9c1c5c9de550..420ca657a339d 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result +++ b/storage/spider/mysql-test/spider/bugfix/r/spider_join_with_non_spider.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t1 (c int); create table t2 (d int); diff --git a/storage/spider/mysql-test/spider/bugfix/r/subquery.result b/storage/spider/mysql-test/spider/bugfix/r/subquery.result index c6ea45e2dc0a4..280f57155bdb8 100644 --- a/storage/spider/mysql-test/spider/bugfix/r/subquery.result +++ b/storage/spider/mysql-test/spider/bugfix/r/subquery.result @@ -4,6 +4,7 @@ for master_1 for child2 for child3 +set spider_same_server_link=1; CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t1 (c1 int); create table t2 (c2 int); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test index f9e157d33e60d..dcf1438fb8088 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_26151.test @@ -14,6 +14,10 @@ --let $srv=srv_mdev_26151 set @old_spider_bgs_mode= @@spider_bgs_mode; set session spider_bgs_mode=1; +set spider_same_server_link=1; +set @old_spider_same_server_link=@@global.spider_same_server_link; +set global spider_same_server_link=1; + evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # casual_read != 0 && casual_read != 1 @@ -42,6 +46,7 @@ drop table td, ts; eval drop server $srv; set session spider_bgs_mode=@old_spider_bgs_mode; +set global spider_same_server_link=@old_spider_same_server_link; --disable_query_log --disable_result_log diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test index 7a011520bb6e7..feff85df6b37e 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28739_simple.test @@ -12,6 +12,7 @@ #set @@global.debug_dbug="d:t:i:o,mysqld.trace"; set global query_cache_type= on; +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t2 (c int); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test index 4f23168ec1b57..a1642f7a9cdbe 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28856.test @@ -9,6 +9,7 @@ # This test covers some table params under consideration for inclusion # in the engine-defined options to be implemented in MDEV-28856. +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test index e1735706341c6..66e3a15addbd1 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_28998.test @@ -12,6 +12,7 @@ if (`select not(count(*)) from information_schema.system_variables where variabl --source ../../t/test_init.inc --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test index ac116fd0e4fef..2e56583d8311a 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29163.test @@ -6,6 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER s FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test index 16291f82075f2..89d53227c6c5d 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29456.test @@ -8,6 +8,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test index ab7a4ded07cdc..93b38c7963c60 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29963.test @@ -7,6 +7,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test index 4dc3dafab24ac..9c59adc80dd02 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30014.test @@ -7,6 +7,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test index 03417013b0749..36e06f3f3c467 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30392.test @@ -6,6 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 (a INT); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test index e628c3b921d64..a3698c9771723 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31338.test @@ -9,6 +9,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t (c BLOB) ENGINE=InnoDB; CREATE TABLE ts (c BLOB) ENGINE=Spider COMMENT='WRAPPER "mysql",srv "srv",TABLE "t"'; diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test index 64cbf4154aa12..a5942fad2b303 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31524.test @@ -10,6 +10,7 @@ --let $srv=srv_mdev_31524 SET @old_spider_read_only_mode = @@session.spider_read_only_mode; +set spider_same_server_link=1; evalp CREATE SERVER $srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); # when the user does not set var nor the table option, the default diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test index bec9dd6c3162b..4dfe3b57e5019 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31645.test @@ -6,6 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); CREATE TABLE t1 ( a bigint(20) NOT NULL, b bigint(20) DEFAULT 0, PRIMARY KEY (a)); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test index 3e823790c8274..93b004a06ebe9 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_31996.test @@ -4,6 +4,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test index 94081d24ad809..144387452f7c2 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_32986.test @@ -6,6 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER mysql OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); diff --git a/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test b/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test index 7b5d38014ed99..294b469a8a559 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test +++ b/storage/spider/mysql-test/spider/bugfix/t/spider_join_with_non_spider.test @@ -8,6 +8,7 @@ --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t1 (c int); create table t2 (d int); diff --git a/storage/spider/mysql-test/spider/bugfix/t/subquery.test b/storage/spider/mysql-test/spider/bugfix/t/subquery.test index 7a50719603d56..70238a524a6ed 100644 --- a/storage/spider/mysql-test/spider/bugfix/t/subquery.test +++ b/storage/spider/mysql-test/spider/bugfix/t/subquery.test @@ -6,6 +6,7 @@ --source ../../t/test_init.inc --enable_result_log --enable_query_log +set spider_same_server_link=1; evalp CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); create table t1 (c1 int); create table t2 (c2 int); diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index d2e9dc1f0ee89..fe83c6522600a 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -1985,7 +1985,7 @@ int spider_db_mbase::connect( if (!spider_param_same_server_link(thd)) { - if (!strcmp(tgt_host, my_localhost)) + if (!strcmp(tgt_host, my_localhost) || !tgt_host || !tgt_host[0]) { if (!strcmp(tgt_socket, *spd_mysqld_unix_port)) { @@ -1995,7 +1995,7 @@ int spider_db_mbase::connect( DBUG_RETURN(ER_SPIDER_SAME_SERVER_LINK_NUM); } } else if (!strcmp(tgt_host, "127.0.0.1") || - !strcmp(tgt_host, glob_hostname)) + !strcmp(tgt_host, glob_hostname) || !tgt_host || !tgt_host[0]) { if (tgt_port == (long) *spd_mysqld_port) { From c4ebf87f862ad6ab610e553a94dfa385a94a116c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 13 Dec 2023 21:02:44 +0100 Subject: [PATCH 12/33] MDEV-32984 Update federated table and column privileges mark auto-inc columns for read/write on INSERT, but only for read on UPDATE --- mysql-test/suite/federated/update.result | 36 ++++++++++++++++++++++++ mysql-test/suite/federated/update.test | 32 +++++++++++++++++++++ sql/log_event_server.cc | 2 +- sql/table.cc | 9 +++--- sql/table.h | 2 +- 5 files changed, 75 insertions(+), 6 deletions(-) create mode 100644 mysql-test/suite/federated/update.result create mode 100644 mysql-test/suite/federated/update.test diff --git a/mysql-test/suite/federated/update.result b/mysql-test/suite/federated/update.result new file mode 100644 index 0000000000000..1905f80ed71f1 --- /dev/null +++ b/mysql-test/suite/federated/update.result @@ -0,0 +1,36 @@ +connect master,127.0.0.1,root,,test,$MASTER_MYPORT,; +connect slave,127.0.0.1,root,,test,$SLAVE_MYPORT,; +connection master; +CREATE DATABASE federated; +connection slave; +CREATE DATABASE federated; +# +# MDEV-32984 Update federated table and column privileges +# +connection slave; +create database db1; +create user my@localhost identified by '1qaz2wsx'; +create table db1.t1 ( +f1 int auto_increment primary key, +f2 varchar(50), +f3 varchar(50), +unique (f2) +); +grant insert, select (f1, f2, f3), update (f3) on db1.t1 to my@localhost; +connection master; +create table tt1 engine=federated connection='mysql://my:1qaz2wsx@localhost:$SLAVE_MYPORT/db1/t1'; +insert into tt1 (f2,f3) values ('test','123'); +select * from tt1; +f1 f2 f3 +1 test 123 +update tt1 set f3='123456' where f2='test'; +drop table tt1; +connection slave; +drop database db1; +drop user my@localhost; +connection master; +DROP TABLE IF EXISTS federated.t1; +DROP DATABASE IF EXISTS federated; +connection slave; +DROP TABLE IF EXISTS federated.t1; +DROP DATABASE IF EXISTS federated; diff --git a/mysql-test/suite/federated/update.test b/mysql-test/suite/federated/update.test new file mode 100644 index 0000000000000..5a0414f1e1a0d --- /dev/null +++ b/mysql-test/suite/federated/update.test @@ -0,0 +1,32 @@ +source include/federated.inc; +source have_federatedx.inc; + +--echo # +--echo # MDEV-32984 Update federated table and column privileges +--echo # +connection slave; +create database db1; +create user my@localhost identified by '1qaz2wsx'; +create table db1.t1 ( + f1 int auto_increment primary key, + f2 varchar(50), + f3 varchar(50), + unique (f2) +); +grant insert, select (f1, f2, f3), update (f3) on db1.t1 to my@localhost; + +connection master; +evalp create table tt1 engine=federated connection='mysql://my:1qaz2wsx@localhost:$SLAVE_MYPORT/db1/t1'; +insert into tt1 (f2,f3) values ('test','123'); +select * from tt1; +update tt1 set f3='123456' where f2='test'; + +drop table tt1; + +connection slave; +drop database db1; +drop user my@localhost; + +source include/federated_cleanup.inc; + + diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index f7a8a1db574f6..13a981d438117 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -7103,7 +7103,7 @@ Write_rows_log_event::do_before_row_operations(const Slave_reporting_capability indexed and it cannot have a DEFAULT value). */ m_table->auto_increment_field_not_null= FALSE; - m_table->mark_auto_increment_column(); + m_table->mark_auto_increment_column(true); } return error; diff --git a/sql/table.cc b/sql/table.cc index 5ed8022da22cd..a5e33052d3f5b 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7320,7 +7320,7 @@ inline void TABLE::mark_index_columns_for_read(uint index) always set and sometimes read. */ -void TABLE::mark_auto_increment_column() +void TABLE::mark_auto_increment_column(bool is_insert) { DBUG_ASSERT(found_next_number_field); /* @@ -7328,7 +7328,8 @@ void TABLE::mark_auto_increment_column() store() to check overflow of auto_increment values */ bitmap_set_bit(read_set, found_next_number_field->field_index); - bitmap_set_bit(write_set, found_next_number_field->field_index); + if (is_insert) + bitmap_set_bit(write_set, found_next_number_field->field_index); if (s->next_number_keypart) mark_index_columns_for_read(s->next_number_index); file->column_bitmaps_signal(); @@ -7453,7 +7454,7 @@ void TABLE::mark_columns_needed_for_update() else { if (found_next_number_field) - mark_auto_increment_column(); + mark_auto_increment_column(false); } if (file->ha_table_flags() & HA_PRIMARY_KEY_REQUIRED_FOR_DELETE) @@ -7527,7 +7528,7 @@ void TABLE::mark_columns_needed_for_insert() triggers->mark_fields_used(TRG_EVENT_INSERT); } if (found_next_number_field) - mark_auto_increment_column(); + mark_auto_increment_column(true); if (default_field) mark_default_fields_for_write(TRUE); /* Mark virtual columns for insert */ diff --git a/sql/table.h b/sql/table.h index 45095a8d1370f..9d659bb71eda4 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1585,7 +1585,7 @@ struct TABLE void mark_index_columns_no_reset(uint index, MY_BITMAP *bitmap); void mark_index_columns_for_read(uint index); void restore_column_maps_after_keyread(MY_BITMAP *backup); - void mark_auto_increment_column(void); + void mark_auto_increment_column(bool insert_fl); void mark_columns_needed_for_update(void); void mark_columns_needed_for_delete(void); void mark_columns_needed_for_insert(void); From 761d5c8987f1770767591ea3bf43f4048ffba8c1 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 10 Jan 2024 00:56:19 +0100 Subject: [PATCH 13/33] MDEV-33092 Undefined reference to concurrency on Solaris remove thr_setconcurrency() followup for 8bbcaab160c Fix by Rainer Orth --- cmake/os/WindowsCache.cmake | 1 - config.h.cmake | 1 - configure.cmake | 1 - include/my_pthread.h | 6 ------ libmysqld/lib_sql.cc | 2 -- mysys/thr_alarm.c | 1 - mysys/thr_lock.c | 3 --- mysys/thr_timer.c | 1 - sql/mysqld.cc | 2 -- storage/maria/unittest/ma_pagecache_consist.c | 4 ---- storage/maria/unittest/ma_pagecache_rwconsist.c | 4 ---- storage/maria/unittest/ma_pagecache_rwconsist2.c | 4 ---- storage/maria/unittest/ma_pagecache_single.c | 4 ---- storage/maria/unittest/ma_test_loghandler_multithread-t.c | 4 ---- 14 files changed, 38 deletions(-) diff --git a/cmake/os/WindowsCache.cmake b/cmake/os/WindowsCache.cmake index 4a3b3383393fd..628b17287475d 100644 --- a/cmake/os/WindowsCache.cmake +++ b/cmake/os/WindowsCache.cmake @@ -243,7 +243,6 @@ SET(HAVE_TERMCAP_H CACHE INTERNAL "") SET(HAVE_TERMIOS_H CACHE INTERNAL "") SET(HAVE_TERMIO_H CACHE INTERNAL "") SET(HAVE_TERM_H CACHE INTERNAL "") -SET(HAVE_THR_SETCONCURRENCY CACHE INTERNAL "") SET(HAVE_THR_YIELD CACHE INTERNAL "") SET(HAVE_TIME 1 CACHE INTERNAL "") SET(HAVE_TIMES CACHE INTERNAL "") diff --git a/config.h.cmake b/config.h.cmake index a8154bb18d1fa..652b95a1ed20a 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -231,7 +231,6 @@ #cmakedefine HAVE_STRTOUL 1 #cmakedefine HAVE_STRTOULL 1 #cmakedefine HAVE_TELL 1 -#cmakedefine HAVE_THR_SETCONCURRENCY 1 #cmakedefine HAVE_THR_YIELD 1 #cmakedefine HAVE_TIME 1 #cmakedefine HAVE_TIMES 1 diff --git a/configure.cmake b/configure.cmake index c7266cdef8109..fcfce85edb9be 100644 --- a/configure.cmake +++ b/configure.cmake @@ -417,7 +417,6 @@ CHECK_FUNCTION_EXISTS (strtoul HAVE_STRTOUL) CHECK_FUNCTION_EXISTS (strtoull HAVE_STRTOULL) CHECK_FUNCTION_EXISTS (strcasecmp HAVE_STRCASECMP) CHECK_FUNCTION_EXISTS (tell HAVE_TELL) -CHECK_FUNCTION_EXISTS (thr_setconcurrency HAVE_THR_SETCONCURRENCY) CHECK_FUNCTION_EXISTS (thr_yield HAVE_THR_YIELD) CHECK_FUNCTION_EXISTS (vasprintf HAVE_VASPRINTF) CHECK_FUNCTION_EXISTS (vsnprintf HAVE_VSNPRINTF) diff --git a/include/my_pthread.h b/include/my_pthread.h index a95e6f77b133a..10cc2301ea691 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -148,9 +148,6 @@ int pthread_cancel(pthread_t thread); #ifndef _REENTRANT #define _REENTRANT #endif -#ifdef HAVE_THR_SETCONCURRENCY -#include /* Probably solaris */ -#endif #ifdef HAVE_SCHED_H #include #endif @@ -619,9 +616,6 @@ extern int my_rw_trywrlock(my_rw_lock_t *); #define GETHOSTBYADDR_BUFF_SIZE 2048 -#ifndef HAVE_THR_SETCONCURRENCY -#define thr_setconcurrency(A) pthread_dummy(0) -#endif #if !defined(HAVE_PTHREAD_ATTR_SETSTACKSIZE) && ! defined(pthread_attr_setstacksize) #define pthread_attr_setstacksize(A,B) pthread_dummy(0) #endif diff --git a/libmysqld/lib_sql.cc b/libmysqld/lib_sql.cc index 8f772ac81223d..41eee607d2312 100644 --- a/libmysqld/lib_sql.cc +++ b/libmysqld/lib_sql.cc @@ -635,8 +635,6 @@ int init_embedded_server(int argc, char **argv, char **groups) udf_init(); #endif - (void) thr_setconcurrency(concurrency); // 10 by default - if (flush_time && flush_time != ~(ulong) 0L) start_handle_manager(); diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 90429f728232c..d2b542ee9b678 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -786,7 +786,6 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) mysql_mutex_unlock(&LOCK_thread_count); DBUG_PRINT("info",("signal thread created")); - thr_setconcurrency(3); pthread_attr_setscope(&thr_attr,PTHREAD_SCOPE_PROCESS); printf("Main thread: %s\n",my_thread_name()); for (i=0 ; i < 2 ; i++) diff --git a/mysys/thr_lock.c b/mysys/thr_lock.c index 168ec035e79d9..c9bc3a2d9fbf1 100644 --- a/mysys/thr_lock.c +++ b/mysys/thr_lock.c @@ -1783,9 +1783,6 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) error,errno); exit(1); } -#endif -#ifdef HAVE_THR_SETCONCURRENCY - (void) thr_setconcurrency(2); #endif for (i=0 ; i < array_elements(lock_counts) ; i++) { diff --git a/mysys/thr_timer.c b/mysys/thr_timer.c index f87c1f755557c..d3627fea9830e 100644 --- a/mysys/thr_timer.c +++ b/mysys/thr_timer.c @@ -533,7 +533,6 @@ static void run_test() mysql_mutex_init(0, &LOCK_thread_count, MY_MUTEX_INIT_FAST); mysql_cond_init(0, &COND_thread_count, NULL); - thr_setconcurrency(3); pthread_attr_init(&thr_attr); pthread_attr_setscope(&thr_attr,PTHREAD_SCOPE_PROCESS); printf("Main thread: %s\n",my_thread_name()); diff --git a/sql/mysqld.cc b/sql/mysqld.cc index b698928c5f7fe..05545a1f4fb23 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5490,8 +5490,6 @@ int mysqld_main(int argc, char **argv) SYSVAR_AUTOSIZE(my_thread_stack_size, new_thread_stack_size); } - (void) thr_setconcurrency(concurrency); // 10 by default - select_thread=pthread_self(); select_thread_in_use=1; diff --git a/storage/maria/unittest/ma_pagecache_consist.c b/storage/maria/unittest/ma_pagecache_consist.c index 29fa29ca0352e..4a9056e66c237 100644 --- a/storage/maria/unittest/ma_pagecache_consist.c +++ b/storage/maria/unittest/ma_pagecache_consist.c @@ -403,10 +403,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, 0)) == 0) { diff --git a/storage/maria/unittest/ma_pagecache_rwconsist.c b/storage/maria/unittest/ma_pagecache_rwconsist.c index a3303eb65a4ef..c4a2148175d17 100644 --- a/storage/maria/unittest/ma_pagecache_rwconsist.c +++ b/storage/maria/unittest/ma_pagecache_rwconsist.c @@ -272,10 +272,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, 0)) == 0) { diff --git a/storage/maria/unittest/ma_pagecache_rwconsist2.c b/storage/maria/unittest/ma_pagecache_rwconsist2.c index 2a0f76b478ff5..e1a80ac495bb0 100644 --- a/storage/maria/unittest/ma_pagecache_rwconsist2.c +++ b/storage/maria/unittest/ma_pagecache_rwconsist2.c @@ -268,10 +268,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, 0)) == 0) { diff --git a/storage/maria/unittest/ma_pagecache_single.c b/storage/maria/unittest/ma_pagecache_single.c index b0a259c85a4ef..a8aecc10a42d9 100644 --- a/storage/maria/unittest/ma_pagecache_single.c +++ b/storage/maria/unittest/ma_pagecache_single.c @@ -795,10 +795,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if ((pagen= init_pagecache(&pagecache, PCACHE_SIZE, 0, 0, TEST_PAGE_SIZE, 0, MYF(MY_WME))) == 0) { diff --git a/storage/maria/unittest/ma_test_loghandler_multithread-t.c b/storage/maria/unittest/ma_test_loghandler_multithread-t.c index cb4d2bc70bae8..e07f8d56e3232 100644 --- a/storage/maria/unittest/ma_test_loghandler_multithread-t.c +++ b/storage/maria/unittest/ma_test_loghandler_multithread-t.c @@ -331,10 +331,6 @@ int main(int argc __attribute__((unused)), exit(1); } -#ifdef HAVE_THR_SETCONCURRENCY - thr_setconcurrency(2); -#endif - if (ma_control_file_open(TRUE, TRUE, TRUE)) { fprintf(stderr, "Can't init control file (%d)\n", errno); From 7801c6d22dce546e9661be909dd0302000933f0b Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Fri, 15 Dec 2023 17:37:28 +1100 Subject: [PATCH 14/33] MDEV-29002 Spider: remove SPIDER_CONN::loop_check_meraged_last The field is assigned but unused, and it causes heap-use-after-free. --- .../spider/bugfix/r/mdev_29002.result | 34 +++++++++++++++++++ .../spider/bugfix/t/mdev_29002.test | 32 +++++++++++++++++ storage/spider/spd_conn.cc | 6 ---- storage/spider/spd_include.h | 1 - 4 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result new file mode 100644 index 0000000000000..894f51c5e3670 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29002.result @@ -0,0 +1,34 @@ +for master_1 +for child2 +for child3 +SET spider_same_server_link= on; +CREATE SERVER s FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +CREATE TABLE t (a INT); +CREATE TABLE t1_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +CREATE TABLE t2_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +SELECT * FROM t1_spider, t2_spider; +a a +SELECT table_name, index_name, cardinality FROM INFORMATION_SCHEMA.STATISTICS WHERE table_name IN ('t1_spider','t2_spider'); +table_name index_name cardinality +RENAME TABLE t1_spider TO t3_spider; +SELECT * FROM t3_spider; +a +DROP TABLE t3_spider, t2_spider, t; +drop server s; +CREATE TABLE t1 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t3 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +SHOW TABLE STATUS; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary +t1 SPIDER 10 NULL 0 0 0 0 0 NULL NULL NULL NULL latin1_swedish_ci NULL Unable to connect to foreign data source: srv 0 +t2 SPIDER 10 NULL 0 0 0 0 0 NULL NULL NULL NULL latin1_swedish_ci NULL Unable to connect to foreign data source: srv 0 +t3 SPIDER 10 NULL 0 0 0 0 0 NULL NULL NULL NULL latin1_swedish_ci NULL Unable to connect to foreign data source: srv 0 +Warnings: +Warning 1429 Unable to connect to foreign data source: srv +Warning 1429 Unable to connect to foreign data source: srv +Warning 1429 Unable to connect to foreign data source: srv +drop table t1, t2, t3; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test new file mode 100644 index 0000000000000..51620a5a23c91 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29002.test @@ -0,0 +1,32 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log +SET spider_same_server_link= on; +evalp CREATE SERVER s FOREIGN DATA WRAPPER mysql +OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); + +CREATE TABLE t (a INT); +CREATE TABLE t1_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +CREATE TABLE t2_spider (a INT) ENGINE=SPIDER COMMENT = "wrapper 'mysql', srv 's', table 't'"; +SELECT * FROM t1_spider, t2_spider; +SELECT table_name, index_name, cardinality FROM INFORMATION_SCHEMA.STATISTICS WHERE table_name IN ('t1_spider','t2_spider'); +RENAME TABLE t1_spider TO t3_spider; +SELECT * FROM t3_spider; + +DROP TABLE t3_spider, t2_spider, t; +drop server s; + +# case by roel +CREATE TABLE t1 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t2 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +CREATE TABLE t3 (c INT) ENGINE=Spider COMMENT='WRAPPER "mysql",HOST "srv",TABLE "t"'; +SHOW TABLE STATUS; +drop table t1, t2, t3; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc index 5ca3bb06fec21..9897e495c8377 100644 --- a/storage/spider/spd_conn.cc +++ b/storage/spider/spd_conn.cc @@ -1783,13 +1783,7 @@ int spider_conn_queue_and_merge_loop_check( lcptr->flag = SPIDER_LOP_CHK_MERAGED; lcptr->next = NULL; if (!conn->loop_check_meraged_first) - { conn->loop_check_meraged_first = lcptr; - conn->loop_check_meraged_last = lcptr; - } else { - conn->loop_check_meraged_last->next = lcptr; - conn->loop_check_meraged_last = lcptr; - } } DBUG_RETURN(0); diff --git a/storage/spider/spd_include.h b/storage/spider/spd_include.h index 59c3a181780f6..8bf44d094b263 100644 --- a/storage/spider/spd_include.h +++ b/storage/spider/spd_include.h @@ -940,7 +940,6 @@ typedef struct st_spider_conn SPIDER_CONN_LOOP_CHECK *loop_check_ignored_first; SPIDER_CONN_LOOP_CHECK *loop_check_ignored_last; SPIDER_CONN_LOOP_CHECK *loop_check_meraged_first; - SPIDER_CONN_LOOP_CHECK *loop_check_meraged_last; } SPIDER_CONN; typedef struct st_spider_lgtm_tblhnd_share From 9e9e0b99ade0e6ccd34abc3bc3abaf7bbd5ecf4e Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 19 Dec 2023 12:18:13 +1100 Subject: [PATCH 15/33] MDEV-30170 ha_spider::delete_table() should report table not exist All Spider tables are recorded in the system table mysql.spider_tables. Deleting a spider table removes the corresponding rows from the system table, among other things. This patch makes it so that if spider could not find any record in the system table to delete for a given table, it should correctly report that no such Spider table exists. --- storage/spider/ha_spider.cc | 9 +++++++- .../spider/bugfix/r/mdev_30170.result | 7 ++++++ .../spider/bugfix/t/mdev_30170.test | 8 +++++++ storage/spider/spd_sys_table.cc | 22 ++++++++++++++++++- 4 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index 08d3fc40f0595..5011f86c044ad 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -11371,7 +11371,10 @@ int ha_spider::create( if ( thd->lex->create_info.or_replace() && (error_num = spider_delete_tables( - table_tables, tmp_share.table_name, &dummy)) + table_tables, tmp_share.table_name, &dummy)) && + /* In this context, no key found in mysql.spider_tables means + the Spider table does not exist */ + error_num != HA_ERR_KEY_NOT_FOUND ) { goto error; } @@ -11842,6 +11845,10 @@ int ha_spider::delete_table( (error_num = spider_delete_tables( table_tables, name, &old_link_count)) ) { + /* In this context, no key found in mysql.spider_tables means + the Spider table does not exist */ + if (error_num == HA_ERR_KEY_NOT_FOUND) + error_num= HA_ERR_NO_SUCH_TABLE; goto error; } spider_close_sys_table(current_thd, table_tables, diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result new file mode 100644 index 0000000000000..2183447bfa375 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_30170.result @@ -0,0 +1,7 @@ +install soname 'ha_spider'; +DROP TABLE non_existing_table; +ERROR 42S02: Unknown table 'test.non_existing_table' +create or replace table non_existing_table (c int) engine=Spider; +drop table non_existing_table; +Warnings: +Warning 1620 Plugin is busy and will be uninstalled on shutdown diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test new file mode 100644 index 0000000000000..690268432d33e --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_30170.test @@ -0,0 +1,8 @@ +install soname 'ha_spider'; +--error ER_BAD_TABLE_ERROR +DROP TABLE non_existing_table; +# Test that create or replace a non existing spider table work +create or replace table non_existing_table (c int) engine=Spider; +drop table non_existing_table; +--disable_query_log +--source ../../include/clean_up_spider.inc diff --git a/storage/spider/spd_sys_table.cc b/storage/spider/spd_sys_table.cc index b3a45b73af773..a8549d5467d96 100644 --- a/storage/spider/spd_sys_table.cc +++ b/storage/spider/spd_sys_table.cc @@ -1865,6 +1865,16 @@ int spider_delete_xa_member( DBUG_RETURN(0); } +/** + Delete a Spider table from mysql.spider_tables. + + @param table The table mysql.spider_tables + @param name The name of the Spider table to delete + @param old_link_count The number of links in the deleted table + + @retval 0 Success + @retval nonzero Failure +*/ int spider_delete_tables( TABLE *table, const char *name, @@ -1880,10 +1890,20 @@ int spider_delete_tables( { spider_store_tables_link_idx(table, roop_count); if ((error_num = spider_check_sys_table(table, table_key))) - break; + { + /* There's a problem with finding the first record for the + spider table, likely because it does not exist. Fail */ + if (roop_count == 0) + DBUG_RETURN(error_num); + /* At least one row has been deleted for the Spider table. + Success */ + else + break; + } else { if ((error_num = spider_delete_sys_table_row(table))) { + /* There's a problem deleting the row. Fail */ DBUG_RETURN(error_num); } } From 9a5f85dcbe12dd9e3228d77834aed8ab53885f20 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Mon, 13 Nov 2023 15:40:13 +0100 Subject: [PATCH 16/33] MDEV-32790: Output result in show create table for mysql_json type should be longtext - We don't test `json` MySQL tables from `std_data` since the error `ER_TABLE_NEEDS_REBUILD ` is invoked. However MDEV-32235 will override this test after merge, but leave it to show behavior and historical changes. - Closes PR #2833 Reviewer: --- .../main/mysql_json_table_recreate.result | 44 +++++++++++++++++++ .../main/mysql_json_table_recreate.test | 36 +++++++++++++++ plugin/type_mysql_json/type.cc | 2 +- 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/mysql_json_table_recreate.result b/mysql-test/main/mysql_json_table_recreate.result index 207dde9d8ad2c..b2ab19f4fbf5f 100644 --- a/mysql-test/main/mysql_json_table_recreate.result +++ b/mysql-test/main/mysql_json_table_recreate.result @@ -169,3 +169,47 @@ Total_Number_of_Tests Succesful_Tests String_is_valid_JSON drop table tempty; drop table mysql_json_test; drop table mysql_json_test_big; +# +# MDEV-32790: Output result in show create table +# for mysql_json type should be longtext +# +create table t1(j json); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL CHECK (json_valid(`j`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +drop table t1; +create table t1(j mysql_json); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` mysql_json /* JSON from MySQL 5.7 */ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +drop table t1; +create table `testjson` ( +`t` json /* JSON from MySQL 5.7*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; +ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL +) ENGINE=InnoDB DEFAULT CH...' at line 2 +create table `testjson` ( +`t` json /* JSON from MySQL 5.7*/ COLLATE utf8mb4_bin NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; +show create table testjson; +Table Create Table +testjson CREATE TABLE `testjson` ( + `t` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL CHECK (json_valid(`t`)) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +drop table testjson; +create table `testjson` ( +`t` longtext /* JSON from MySQL 5.7 */ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; +show create table testjson; +Table Create Table +testjson CREATE TABLE `testjson` ( + `t` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +drop table testjson; +# +# End of 10.5 tests +# diff --git a/mysql-test/main/mysql_json_table_recreate.test b/mysql-test/main/mysql_json_table_recreate.test index a399b546591b2..7e9895a1f107f 100644 --- a/mysql-test/main/mysql_json_table_recreate.test +++ b/mysql-test/main/mysql_json_table_recreate.test @@ -88,3 +88,39 @@ from mysql_json_test_big; drop table tempty; drop table mysql_json_test; drop table mysql_json_test_big; + +--echo # +--echo # MDEV-32790: Output result in show create table +--echo # for mysql_json type should be longtext +--echo # + +create table t1(j json); +show create table t1; +drop table t1; +create table t1(j mysql_json); +show create table t1; +drop table t1; +# `json` type should not have character set and collation other than utf8mb4_bin +--error ER_PARSE_ERROR +create table `testjson` ( + `t` json /* JSON from MySQL 5.7*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; + +# By removing character set from `json` field query should work and +# expand to `longtext` with characterset +create table `testjson` ( + `t` json /* JSON from MySQL 5.7*/ COLLATE utf8mb4_bin NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; +show create table testjson; +drop table testjson; + +# `longtext` that is alias can have character set +create table `testjson` ( + `t` longtext /* JSON from MySQL 5.7 */ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; +show create table testjson; +drop table testjson; + +--echo # +--echo # End of 10.5 tests +--echo # diff --git a/plugin/type_mysql_json/type.cc b/plugin/type_mysql_json/type.cc index c5168969b67e2..c915e0312bbc3 100644 --- a/plugin/type_mysql_json/type.cc +++ b/plugin/type_mysql_json/type.cc @@ -62,7 +62,7 @@ class Field_mysql_json: public Field_blob bool parse_mysql(String *dest, const char *data, size_t length) const; bool send(Protocol *protocol) { return Field::send(protocol); } void sql_type(String &s) const - { s.set_ascii(STRING_WITH_LEN("json /* MySQL 5.7 */")); } + { s.set_ascii(STRING_WITH_LEN("mysql_json /* JSON from MySQL 5.7 */")); } /* this will make ALTER TABLE to consider it different from built-in field */ Compression_method *compression_method() const { return (Compression_method*)1; } }; From 22f3ebe4bf11acb7b5e854bc4e6b3439af6982f1 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 11 Jan 2024 15:15:46 +0100 Subject: [PATCH 17/33] MDEV-32235: mysql_json cannot be used on newly created table Closes PR #2839 Reviewer: cvicentiu@mariadb.org --- .../main/mysql_json_table_recreate.result | 18 ++++++++++++------ mysql-test/main/mysql_json_table_recreate.test | 17 +++++++++++++++-- sql/sql_table.cc | 13 +++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/mysql-test/main/mysql_json_table_recreate.result b/mysql-test/main/mysql_json_table_recreate.result index b2ab19f4fbf5f..a0c2483d3d924 100644 --- a/mysql-test/main/mysql_json_table_recreate.result +++ b/mysql-test/main/mysql_json_table_recreate.result @@ -181,12 +181,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table t1; create table t1(j mysql_json); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `j` mysql_json /* JSON from MySQL 5.7 */ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL -) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -drop table t1; +ERROR HY000: Cannot create table `test`.`t1`: Run mariadb-upgrade, to upgrade table with mysql_json type. create table `testjson` ( `t` json /* JSON from MySQL 5.7*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; @@ -211,5 +206,16 @@ testjson CREATE TABLE `testjson` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table testjson; # +# MDEV-32235: mysql_json cannot be used on newly created table +# +CREATE TABLE t(j mysql_json); +ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. +CREATE TABLE IF NOT EXISTS t(j mysql_json); +ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. +CREATE OR REPLACE TABLE t(j mysql_json); +ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. +CREATE TEMPORARY TABLE t(j mysql_json); +ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. +# # End of 10.5 tests # diff --git a/mysql-test/main/mysql_json_table_recreate.test b/mysql-test/main/mysql_json_table_recreate.test index 7e9895a1f107f..bf4cac0f90955 100644 --- a/mysql-test/main/mysql_json_table_recreate.test +++ b/mysql-test/main/mysql_json_table_recreate.test @@ -97,9 +97,8 @@ drop table mysql_json_test_big; create table t1(j json); show create table t1; drop table t1; +--error ER_CANT_CREATE_TABLE create table t1(j mysql_json); -show create table t1; -drop table t1; # `json` type should not have character set and collation other than utf8mb4_bin --error ER_PARSE_ERROR create table `testjson` ( @@ -121,6 +120,20 @@ create table `testjson` ( show create table testjson; drop table testjson; + +--echo # +--echo # MDEV-32235: mysql_json cannot be used on newly created table +--echo # + +--error ER_CANT_CREATE_TABLE +CREATE TABLE t(j mysql_json); +--error ER_CANT_CREATE_TABLE +CREATE TABLE IF NOT EXISTS t(j mysql_json); +--error ER_CANT_CREATE_TABLE +CREATE OR REPLACE TABLE t(j mysql_json); +--error ER_CANT_CREATE_TABLE +CREATE TEMPORARY TABLE t(j mysql_json); + --echo # --echo # End of 10.5 tests --echo # diff --git a/sql/sql_table.cc b/sql/sql_table.cc index d68bbd1ac4f69..8088d3e0d70e3 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3657,6 +3657,19 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (sql_field->vcol_info) sql_field->flags&= ~NOT_NULL_FLAG; + if (sql_field->real_field_type() == MYSQL_TYPE_BLOB && + thd->lex->sql_command == SQLCOM_CREATE_TABLE) + { + if (!strcmp(sql_field->type_handler()->name().ptr(), "MYSQL_JSON")) + { + my_printf_error(ER_CANT_CREATE_TABLE, + "Cannot create table %`s.%`s: " + "Run mariadb-upgrade, " + "to upgrade table with mysql_json type.", MYF(0), + alter_info->db.str, alter_info->table_name.str); + DBUG_RETURN(TRUE); + } + } /* Initialize length from its original value (number of characters), which was set in the parser. This is necessary if we're From d0c80c211c1fe3370b68be540bb9113028c6746f Mon Sep 17 00:00:00 2001 From: Dave Gosselin Date: Mon, 8 Jan 2024 11:58:29 -0500 Subject: [PATCH 18/33] MDEV-32090 Test for null-safe equals in join This ticket is fixed by MDEV-32555 and this test captures a different use case. --- mysql-test/main/subselect_nulls_innodb.result | 27 ++++++++++++++++ mysql-test/main/subselect_nulls_innodb.test | 32 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 mysql-test/main/subselect_nulls_innodb.result create mode 100644 mysql-test/main/subselect_nulls_innodb.test diff --git a/mysql-test/main/subselect_nulls_innodb.result b/mysql-test/main/subselect_nulls_innodb.result new file mode 100644 index 0000000000000..2cab41760b314 --- /dev/null +++ b/mysql-test/main/subselect_nulls_innodb.result @@ -0,0 +1,27 @@ +# +# MDEV-32090 Index does not handle null-safe equals operator correctly in join +# +CREATE TEMPORARY TABLE t1 ( +`id` int(10) unsigned NOT NULL, +`number` int(10) unsigned DEFAULT 0, +`name` varchar(47) DEFAULT NULL, +`street` mediumint(8) unsigned DEFAULT NULL, +PRIMARY KEY (`id`), +KEY `streetNumber` (`street`,`number`,`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; +INSERT INTO t1 (id, number, name, street) VALUES (100733476, 14, NULL, 1115569); +SELECT +b1.id +FROM +t1 b1 +INNER JOIN t1 b2 ON ( +b1.street = b2.street +AND b1.number <=> b2.number +AND b1.name <=> b2.name +); +id +100733476 +DROP TABLE t1; +# +# End of 10.11 tests +# diff --git a/mysql-test/main/subselect_nulls_innodb.test b/mysql-test/main/subselect_nulls_innodb.test new file mode 100644 index 0000000000000..79d572a2e0e3b --- /dev/null +++ b/mysql-test/main/subselect_nulls_innodb.test @@ -0,0 +1,32 @@ +--source include/have_innodb.inc + +--echo # +--echo # MDEV-32090 Index does not handle null-safe equals operator correctly in join +--echo # + +CREATE TEMPORARY TABLE t1 ( + `id` int(10) unsigned NOT NULL, + `number` int(10) unsigned DEFAULT 0, + `name` varchar(47) DEFAULT NULL, + `street` mediumint(8) unsigned DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `streetNumber` (`street`,`number`,`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_unicode_ci; + +INSERT INTO t1 (id, number, name, street) VALUES (100733476, 14, NULL, 1115569); + +SELECT + b1.id +FROM + t1 b1 + INNER JOIN t1 b2 ON ( + b1.street = b2.street + AND b1.number <=> b2.number + AND b1.name <=> b2.name + ); + +DROP TABLE t1; + +--echo # +--echo # End of 10.11 tests +--echo # From 8b5c1d5afa33521d7a5024b58a7b8197e60bfd33 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 12 Jan 2024 15:20:25 +0100 Subject: [PATCH 19/33] Revert "MDEV-32235: mysql_json cannot be used on newly created table" This reverts commit 22f3ebe4bf11acb7b5e854bc4e6b3439af6982f1. --- .../main/mysql_json_table_recreate.result | 18 ++++++------------ mysql-test/main/mysql_json_table_recreate.test | 17 ++--------------- sql/sql_table.cc | 13 ------------- 3 files changed, 8 insertions(+), 40 deletions(-) diff --git a/mysql-test/main/mysql_json_table_recreate.result b/mysql-test/main/mysql_json_table_recreate.result index a0c2483d3d924..b2ab19f4fbf5f 100644 --- a/mysql-test/main/mysql_json_table_recreate.result +++ b/mysql-test/main/mysql_json_table_recreate.result @@ -181,7 +181,12 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table t1; create table t1(j mysql_json); -ERROR HY000: Cannot create table `test`.`t1`: Run mariadb-upgrade, to upgrade table with mysql_json type. +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `j` mysql_json /* JSON from MySQL 5.7 */ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +drop table t1; create table `testjson` ( `t` json /* JSON from MySQL 5.7*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; @@ -206,16 +211,5 @@ testjson CREATE TABLE `testjson` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table testjson; # -# MDEV-32235: mysql_json cannot be used on newly created table -# -CREATE TABLE t(j mysql_json); -ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. -CREATE TABLE IF NOT EXISTS t(j mysql_json); -ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. -CREATE OR REPLACE TABLE t(j mysql_json); -ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. -CREATE TEMPORARY TABLE t(j mysql_json); -ERROR HY000: Cannot create table `test`.`t`: Run mariadb-upgrade, to upgrade table with mysql_json type. -# # End of 10.5 tests # diff --git a/mysql-test/main/mysql_json_table_recreate.test b/mysql-test/main/mysql_json_table_recreate.test index bf4cac0f90955..7e9895a1f107f 100644 --- a/mysql-test/main/mysql_json_table_recreate.test +++ b/mysql-test/main/mysql_json_table_recreate.test @@ -97,8 +97,9 @@ drop table mysql_json_test_big; create table t1(j json); show create table t1; drop table t1; ---error ER_CANT_CREATE_TABLE create table t1(j mysql_json); +show create table t1; +drop table t1; # `json` type should not have character set and collation other than utf8mb4_bin --error ER_PARSE_ERROR create table `testjson` ( @@ -120,20 +121,6 @@ create table `testjson` ( show create table testjson; drop table testjson; - ---echo # ---echo # MDEV-32235: mysql_json cannot be used on newly created table ---echo # - ---error ER_CANT_CREATE_TABLE -CREATE TABLE t(j mysql_json); ---error ER_CANT_CREATE_TABLE -CREATE TABLE IF NOT EXISTS t(j mysql_json); ---error ER_CANT_CREATE_TABLE -CREATE OR REPLACE TABLE t(j mysql_json); ---error ER_CANT_CREATE_TABLE -CREATE TEMPORARY TABLE t(j mysql_json); - --echo # --echo # End of 10.5 tests --echo # diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 8088d3e0d70e3..d68bbd1ac4f69 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -3657,19 +3657,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info, if (sql_field->vcol_info) sql_field->flags&= ~NOT_NULL_FLAG; - if (sql_field->real_field_type() == MYSQL_TYPE_BLOB && - thd->lex->sql_command == SQLCOM_CREATE_TABLE) - { - if (!strcmp(sql_field->type_handler()->name().ptr(), "MYSQL_JSON")) - { - my_printf_error(ER_CANT_CREATE_TABLE, - "Cannot create table %`s.%`s: " - "Run mariadb-upgrade, " - "to upgrade table with mysql_json type.", MYF(0), - alter_info->db.str, alter_info->table_name.str); - DBUG_RETURN(TRUE); - } - } /* Initialize length from its original value (number of characters), which was set in the parser. This is necessary if we're From 8a763c014ede4adad9c84852269b5af61845c0d9 Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Fri, 12 Jan 2024 15:39:38 +0100 Subject: [PATCH 20/33] MDEV-32235: mysql_json cannot be used on newly created table - Closes PR #2839 - Usage of `Column_definition_fix_attributes()` suggested by Alexandar Barkov - thanks bar, that is better than hook in server code (reverted 22f3ebe4bf11) - This method is called after parsing the data type: * in `CREATE/ALTER TABLE` * in SP: return data type, parameter data type, variable data type - We want to disallow all these use cases of MYSQL_JSON. - Reviewer: bar@mariadb.com cvicentiu@mariadb.org --- .../main/mysql_json_table_recreate.result | 38 ++++++++++++++--- .../main/mysql_json_table_recreate.test | 41 ++++++++++++++++++- plugin/type_mysql_json/type.cc | 5 +++ 3 files changed, 76 insertions(+), 8 deletions(-) diff --git a/mysql-test/main/mysql_json_table_recreate.result b/mysql-test/main/mysql_json_table_recreate.result index b2ab19f4fbf5f..a61377fe21d57 100644 --- a/mysql-test/main/mysql_json_table_recreate.result +++ b/mysql-test/main/mysql_json_table_recreate.result @@ -30,6 +30,12 @@ show create table mysql_json_test; ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it! select * from mysql_json_test; ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it! +CREATE TABLE t2 AS SELECT * FROM mysql_json_test; +ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it! +CREATE TABLE t2 (a mysql_json /*new column*/) AS SELECT * FROM mysql_json_test; +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE TABLE t2 (actual mysql_json /*existing column*/) AS SELECT * FROM mysql_json_test; +ERROR HY000: 'MYSQL_JSON' is not allowed in this context LOCK TABLES mysql_json_test WRITE; ERROR HY000: Table rebuild required. Please do "ALTER TABLE `test.mysql_json_test` FORCE" or dump/reload to fix it! alter table mysql_json_test force; @@ -181,12 +187,7 @@ t1 CREATE TABLE `t1` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table t1; create table t1(j mysql_json); -show create table t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `j` mysql_json /* JSON from MySQL 5.7 */ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin DEFAULT NULL -) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci -drop table t1; +ERROR HY000: 'MYSQL_JSON' is not allowed in this context create table `testjson` ( `t` json /* JSON from MySQL 5.7*/ CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci; @@ -211,5 +212,30 @@ testjson CREATE TABLE `testjson` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci drop table testjson; # +# MDEV-32235: mysql_json cannot be used on newly created table +# +CREATE TABLE t(j mysql_json); +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE TABLE IF NOT EXISTS t(j mysql_json); +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE OR REPLACE TABLE t(j mysql_json); +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE TEMPORARY TABLE t(j mysql_json); +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE TABLE t1 (a TEXT); +ALTER TABLE t1 MODIFY a mysql_json; +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +DROP TABLE t1; +CREATE FUNCTION f1() RETURNS mysql_json RETURN NULL; +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE FUNCTION f1(a mysql_json) RETURNS INT RETURN 0; +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +CREATE PROCEDURE p1() +BEGIN +DECLARE a mysql_json; +END; +$$ +ERROR HY000: 'MYSQL_JSON' is not allowed in this context +# # End of 10.5 tests # diff --git a/mysql-test/main/mysql_json_table_recreate.test b/mysql-test/main/mysql_json_table_recreate.test index 7e9895a1f107f..a6f1d3194aeb1 100644 --- a/mysql-test/main/mysql_json_table_recreate.test +++ b/mysql-test/main/mysql_json_table_recreate.test @@ -51,6 +51,13 @@ show create table mysql_json_test; --error ER_TABLE_NEEDS_REBUILD select * from mysql_json_test; +--error ER_TABLE_NEEDS_REBUILD +CREATE TABLE t2 AS SELECT * FROM mysql_json_test; +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE TABLE t2 (a mysql_json /*new column*/) AS SELECT * FROM mysql_json_test; +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE TABLE t2 (actual mysql_json /*existing column*/) AS SELECT * FROM mysql_json_test; + --error ER_TABLE_NEEDS_REBUILD LOCK TABLES mysql_json_test WRITE; @@ -97,9 +104,8 @@ drop table mysql_json_test_big; create table t1(j json); show create table t1; drop table t1; +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT create table t1(j mysql_json); -show create table t1; -drop table t1; # `json` type should not have character set and collation other than utf8mb4_bin --error ER_PARSE_ERROR create table `testjson` ( @@ -121,6 +127,37 @@ create table `testjson` ( show create table testjson; drop table testjson; +--echo # +--echo # MDEV-32235: mysql_json cannot be used on newly created table +--echo # + +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE TABLE t(j mysql_json); +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE TABLE IF NOT EXISTS t(j mysql_json); +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE OR REPLACE TABLE t(j mysql_json); +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE TEMPORARY TABLE t(j mysql_json); + +CREATE TABLE t1 (a TEXT); +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +ALTER TABLE t1 MODIFY a mysql_json; +DROP TABLE t1; + +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE FUNCTION f1() RETURNS mysql_json RETURN NULL; +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE FUNCTION f1(a mysql_json) RETURNS INT RETURN 0; +DELIMITER $$; +--error ER_NOT_ALLOWED_IN_THIS_CONTEXT +CREATE PROCEDURE p1() +BEGIN + DECLARE a mysql_json; +END; +$$ +DELIMITER ;$$ + --echo # --echo # End of 10.5 tests --echo # diff --git a/plugin/type_mysql_json/type.cc b/plugin/type_mysql_json/type.cc index c915e0312bbc3..f84f76381a09a 100644 --- a/plugin/type_mysql_json/type.cc +++ b/plugin/type_mysql_json/type.cc @@ -37,6 +37,11 @@ class Type_handler_mysql_json: public Type_handler_blob Field *make_table_field(MEM_ROOT *, const LEX_CSTRING *, const Record_addr &, const Type_all_attributes &, TABLE_SHARE *) const override; + bool Column_definition_fix_attributes(Column_definition *c) const override + { + my_error(ER_NOT_ALLOWED_IN_THIS_CONTEXT, MYF(0), "MYSQL_JSON"); + return true; + } void Column_definition_reuse_fix_attributes(THD *thd, Column_definition *def, const Field *field) const override; From 5b0a4159ef1717545ccd37c08db0cba0eef00e5a Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Fri, 5 Jan 2024 13:44:49 +0100 Subject: [PATCH 21/33] Fix test failures on s390x in test following main.column_compression_rpl The problem is the test is skipped after sourcing include/master-slave.inc. This leaves the slave threads running after the test is skipped, causing a following test to fail during rpl setup. Also rename have_normal_bzip.inc to the more appropriate _zlib. Signed-off-by: Kristian Nielsen --- .../include/{have_normal_bzip.inc => have_normal_zlib.inc} | 4 ++-- mysql-test/main/column_compression.test | 2 +- mysql-test/main/column_compression_rpl.test | 2 +- mysql-test/main/func_compress.test | 2 +- mysql-test/main/mysqlbinlog_row_compressed.test | 2 +- mysql-test/main/mysqlbinlog_stmt_compressed.test | 2 +- mysql-test/suite/compat/oracle/t/column_compression.test | 2 +- mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) rename mysql-test/include/{have_normal_bzip.inc => have_normal_zlib.inc} (67%) diff --git a/mysql-test/include/have_normal_bzip.inc b/mysql-test/include/have_normal_zlib.inc similarity index 67% rename from mysql-test/include/have_normal_bzip.inc rename to mysql-test/include/have_normal_zlib.inc index 36c06274398c8..a4531e687bcec 100644 --- a/mysql-test/include/have_normal_bzip.inc +++ b/mysql-test/include/have_normal_zlib.inc @@ -1,9 +1,9 @@ --source include/have_compress.inc -# Test that the system is using the default/standard bzip library. +# Test that the system is using the default/standard zlib library. # If not, we have to skip the test as the compression lengths displayed # in the test will not match the results from used compression library. if (`select length(COMPRESS(space(5000))) != 33`) { - skip Test skipped as standard bzip is needed; + skip Test skipped as standard zlib is needed; } diff --git a/mysql-test/main/column_compression.test b/mysql-test/main/column_compression.test index c628e8f9cca86..de59cf5c7c82a 100644 --- a/mysql-test/main/column_compression.test +++ b/mysql-test/main/column_compression.test @@ -1,6 +1,6 @@ --source include/have_innodb.inc --source include/have_csv.inc ---source include/have_normal_bzip.inc +--source include/have_normal_zlib.inc let $MYSQLD_DATADIR= `select @@datadir`; diff --git a/mysql-test/main/column_compression_rpl.test b/mysql-test/main/column_compression_rpl.test index df8e889016b8b..18992f33f3c76 100644 --- a/mysql-test/main/column_compression_rpl.test +++ b/mysql-test/main/column_compression_rpl.test @@ -1,6 +1,6 @@ --source include/have_innodb.inc +--source include/have_normal_zlib.inc --source include/master-slave.inc ---source include/have_normal_bzip.inc --let $engine_type= myisam --let $engine_type2= innodb diff --git a/mysql-test/main/func_compress.test b/mysql-test/main/func_compress.test index 221dddd59f558..d18af140ed385 100644 --- a/mysql-test/main/func_compress.test +++ b/mysql-test/main/func_compress.test @@ -1,5 +1,5 @@ -- source include/have_compress.inc --- source include/have_normal_bzip.inc +-- source include/have_normal_zlib.inc # # Test for compress and uncompress functions: # diff --git a/mysql-test/main/mysqlbinlog_row_compressed.test b/mysql-test/main/mysqlbinlog_row_compressed.test index f28068618304c..f493c4db2cd60 100644 --- a/mysql-test/main/mysqlbinlog_row_compressed.test +++ b/mysql-test/main/mysqlbinlog_row_compressed.test @@ -4,7 +4,7 @@ --source include/have_log_bin.inc --source include/have_binlog_format_row.inc ---source include/have_normal_bzip.inc +--source include/have_normal_zlib.inc # # diff --git a/mysql-test/main/mysqlbinlog_stmt_compressed.test b/mysql-test/main/mysqlbinlog_stmt_compressed.test index 400f21f5ab926..1f1591e7a00d5 100644 --- a/mysql-test/main/mysqlbinlog_stmt_compressed.test +++ b/mysql-test/main/mysqlbinlog_stmt_compressed.test @@ -4,7 +4,7 @@ --source include/have_log_bin.inc --source include/have_binlog_format_statement.inc ---source include/have_normal_bzip.inc +--source include/have_normal_zlib.inc # # # mysqlbinlog: compressed query event diff --git a/mysql-test/suite/compat/oracle/t/column_compression.test b/mysql-test/suite/compat/oracle/t/column_compression.test index 01d4977ba961f..e8d55000bc62e 100644 --- a/mysql-test/suite/compat/oracle/t/column_compression.test +++ b/mysql-test/suite/compat/oracle/t/column_compression.test @@ -1,6 +1,6 @@ --source include/have_innodb.inc --source include/have_csv.inc ---source include/have_normal_bzip.inc +--source include/have_normal_zlib.inc SET sql_mode=ORACLE; diff --git a/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test b/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test index dab9bcfa86429..24029a48aa430 100644 --- a/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test +++ b/mysql-test/suite/innodb/t/row_size_error_log_warnings_3.test @@ -1,7 +1,7 @@ --source include/have_innodb.inc --source include/have_sequence.inc --source include/innodb_page_size_small.inc ---source include/have_normal_bzip.inc +--source include/have_normal_zlib.inc call mtr.add_suppression("InnoDB: Cannot add field .* in table .* because after adding it, the row size is .* which is greater than maximum allowed size (.*) for a record on index leaf page."); From 48e4962c44fa5cbdafb6d1ed9564f308a57eeedc Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Thu, 6 Jul 2023 11:55:40 +0700 Subject: [PATCH 22/33] MDEV-29298 INSERT ... SELECT Does not produce an optimizer trace Add INSERT ... SELECT to the list of commands that can be traced Approved by Sergei Petrunia (sergey@mariadb.com) --- mysql-test/main/opt_trace.result | 12 ++++++++++++ mysql-test/main/opt_trace.test | 14 ++++++++++++++ sql/opt_trace.cc | 3 ++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index 5766b8ead4231..66f081c65904c 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -9231,5 +9231,17 @@ json_detailed(json_extract(trace, '$**.in_to_subquery_conversion')) ] set in_predicate_conversion_threshold=@tmp; drop table t0; +# +# MDEV-29298: INSERT ... SELECT Does not produce an optimizer trace +# +create table t1 (a int, b int); +create table t2 (a int, b int); +insert into t1 values (1,1), (2,2), (3,3), (4,4), (5,5); +set optimizer_trace=1; +insert into t2 select * from t1 where a<= b and a>4; +select QUERY, LENGTH(trace)>1 from information_schema.optimizer_trace; +QUERY LENGTH(trace)>1 +insert into t2 select * from t1 where a<= b and a>4 1 +drop table t1, t2; # End of 10.5 tests set optimizer_trace='enabled=off'; diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test index 43539fc764bc5..e7851fc62c57f 100644 --- a/mysql-test/main/opt_trace.test +++ b/mysql-test/main/opt_trace.test @@ -861,5 +861,19 @@ set in_predicate_conversion_threshold=@tmp; drop table t0; --enable_view_protocol +--echo # +--echo # MDEV-29298: INSERT ... SELECT Does not produce an optimizer trace +--echo # +create table t1 (a int, b int); +create table t2 (a int, b int); +insert into t1 values (1,1), (2,2), (3,3), (4,4), (5,5); +set optimizer_trace=1; + +insert into t2 select * from t1 where a<= b and a>4; + +select QUERY, LENGTH(trace)>1 from information_schema.optimizer_trace; + +drop table t1, t2; + --echo # End of 10.5 tests set optimizer_trace='enabled=off'; diff --git a/sql/opt_trace.cc b/sql/opt_trace.cc index ddec6d5ed2df3..4bd545495328f 100644 --- a/sql/opt_trace.cc +++ b/sql/opt_trace.cc @@ -103,7 +103,8 @@ inline bool sql_command_can_be_traced(enum enum_sql_command sql_command) sql_command == SQLCOM_UPDATE || sql_command == SQLCOM_DELETE || sql_command == SQLCOM_DELETE_MULTI || - sql_command == SQLCOM_UPDATE_MULTI; + sql_command == SQLCOM_UPDATE_MULTI || + sql_command == SQLCOM_INSERT_SELECT; } void opt_trace_print_expanded_query(THD *thd, SELECT_LEX *select_lex, From 82f27ea5a49798d5d89479db8715faa71f2a87ab Mon Sep 17 00:00:00 2001 From: Anel Husakovic Date: Thu, 11 Jan 2024 12:54:16 +0100 Subject: [PATCH 23/33] MDEV-33187: Make mariadb-hotcopy compatible with DBI:MariaDB --- scripts/mysqlhotcopy.sh | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/scripts/mysqlhotcopy.sh b/scripts/mysqlhotcopy.sh index 44abcfec0553d..ebf82e46e039b 100644 --- a/scripts/mysqlhotcopy.sh +++ b/scripts/mysqlhotcopy.sh @@ -189,21 +189,38 @@ $opt{quiet} = 0 if $opt{debug}; $opt{allowold} = 1 if $opt{keepold}; # --- connect to the database --- +## Socket takes precedence. my $dsn; -$dsn = ";host=" . (defined($opt{host}) ? $opt{host} : "localhost"); -$dsn .= ";port=$opt{port}" if $opt{port}; -$dsn .= ";mariadb_socket=$opt{socket}" if $opt{socket}; +my $prefix= 'mysql'; -# use mariadb_read_default_group=mysqlhotcopy so that [client] and -# [mysqlhotcopy] groups will be read from standard options files. +if (eval {DBI->install_driver("MariaDB")}) { + $dsn ="DBI:MariaDB:;"; + $prefix= 'mariadb'; +} +else { + $dsn = "DBI:mysql:;"; +} -my $dbh = DBI->connect("DBI:MariaDB:$dsn;mariadb_read_default_group=mysqlhotcopy", - $opt{user}, $opt{password}, +if ($opt{socket} and -S $opt{socket}) +{ + $dsn .= "${prefix}_socket=$opt{socket}"; +} +else { - RaiseError => 1, - PrintError => 0, - AutoCommit => 1, -}); + $dsn .= "host=" . $opt{host}; + if ($opt{host} ne "localhost") + { + $dsn .= ";port=". $opt{port}; + } +} + +$dsn .= ";mariadb_read_default_group=mysqlhotcopy"; + +# use mariadb_read_default_group=mysqlhotcopy so that [client] and +# [mysqlhotcopy] groups will be read from standard options files. +# make the connection to MariaDB +my $dbh= DBI->connect($dsn, $opt{user}, $opt{password}, { RaiseError => 1, PrintError => 0}) || + die("Can't make a connection to the MariaDB server.\n The error: $DBI::errstr"); # --- check that checkpoint table exists if specified --- if ( $opt{checkpoint} ) { From 7702e481df4a4ccce84558dbebccbc5a2fc64fd1 Mon Sep 17 00:00:00 2001 From: Paul Szabo Date: Fri, 12 Jan 2024 16:02:02 +0100 Subject: [PATCH 24/33] MDEV-30259: mariadb-hotcopy fails for performance_schema Signed-off-by: Paul Szabo psz@maths.usyd.edu.au www.maths.usyd.edu.au/u/psz School of Mathematics and Statistics University of Sydney Australia --- scripts/mysqlhotcopy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/mysqlhotcopy.sh b/scripts/mysqlhotcopy.sh index ebf82e46e039b..78399ac3599f5 100644 --- a/scripts/mysqlhotcopy.sh +++ b/scripts/mysqlhotcopy.sh @@ -288,6 +288,7 @@ if ( defined $opt{regexp} ) { $sth_dbs->execute; while ( my ($db_name) = $sth_dbs->fetchrow_array ) { next if $db_name =~ m/^information_schema$/i; + next if $db_name =~ m/^performance_schema$/i; push @db_desc, { 'src' => $db_name, 't_regex' => $t_regex } if ( $db_name =~ m/$opt{regexp}/o ); } } From 78ea9ee4f26d7927cfcadb482a0031e2e40c7684 Mon Sep 17 00:00:00 2001 From: Paul Szabo Date: Fri, 12 Jan 2024 16:03:41 +0100 Subject: [PATCH 25/33] MDEV-33187: mariadb-hotcopy fails for sys Signed-off-by: Paul Szabo psz@maths.usyd.edu.au www.maths.usyd.edu.au/u/psz School of Mathematics and Statistics University of Sydney Australia --- scripts/mysqlhotcopy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/mysqlhotcopy.sh b/scripts/mysqlhotcopy.sh index 78399ac3599f5..d0821e663c4a6 100644 --- a/scripts/mysqlhotcopy.sh +++ b/scripts/mysqlhotcopy.sh @@ -289,6 +289,7 @@ if ( defined $opt{regexp} ) { while ( my ($db_name) = $sth_dbs->fetchrow_array ) { next if $db_name =~ m/^information_schema$/i; next if $db_name =~ m/^performance_schema$/i; + next if $db_name =~ m/^sys$/i; push @db_desc, { 'src' => $db_name, 't_regex' => $t_regex } if ( $db_name =~ m/$opt{regexp}/o ); } } From ee30491e508e6c8c19c899f713662ec2e5042aaa Mon Sep 17 00:00:00 2001 From: Tuukka Pasanen Date: Wed, 15 Nov 2023 11:09:26 +0200 Subject: [PATCH 26/33] MDEV-32111: Debian Sid/Trixie will not have libncurses 5 anymore Upstream Debian Sid which will become Debian Trixie (13) have dropped NCurses version 5 and changed dev package name just libncurses-dev --- debian/control | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/control b/debian/control index 2bbd71d2ac92d..eb6a5cba16cfa 100644 --- a/debian/control +++ b/debian/control @@ -26,8 +26,7 @@ Build-Depends: bison, libjudy-dev, libkrb5-dev, liblz4-dev, - libncurses5-dev (>= 5.0-6~), - libncurses5-dev:native (>= 5.0-6~), + libncurses-dev, libnuma-dev [linux-any], libpam0g-dev, libpcre2-dev, From caad34df549b91b654c69bf3b5644277be783d31 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 15 Jan 2024 14:08:27 +0530 Subject: [PATCH 27/33] MDEV-32968 InnoDB fails to restore tablespace first page from doublewrite buffer when page is empty - InnoDB fails to find the space id from the page0 of the tablespace. In that case, InnoDB can use doublewrite buffer to recover the page0 and write into the file. - buf_dblwr_t::init_or_load_pages(): Loads only the pages which are valid.(page lsn >= checkpoint). To do that, InnoDB has to open the redo log before system tablespace, read the latest checkpoint information. recv_dblwr_t::find_first_page(): 1) Iterate the doublewrite buffer pages and find the 0th page 2) Read the tablespace flags, space id from the 0th page. 3) Read the 1st, 2nd and 3rd page from tablespace file and compare the space id with the space id which is stored in doublewrite buffer. 4) If it matches then we can write into the file. 5) Return space which matches the pages from the file. SysTablespace::read_lsn_and_check_flags(): Remove the retry logic for validating the first page. After restoring the first page from doublewrite buffer, assign tablespace flags by reading the first page. recv_recovery_read_max_checkpoint(): Reads the maximum checkpoint information from log file recv_recovery_from_checkpoint_start(): Avoid reading the checkpoint header information from log file Datafile::validate_first_page(): Throw error in case of first page validation fails. --- mysql-test/suite/innodb/r/doublewrite.result | 25 +++- .../suite/innodb/r/log_file_name.result | 7 +- mysql-test/suite/innodb/t/doublewrite.test | 38 ++++- .../suite/innodb/t/doublewrite_debug.test | 1 + mysql-test/suite/innodb/t/log_file_name.test | 9 +- storage/innobase/buf/buf0dblwr.cc | 11 +- storage/innobase/fsp/fsp0file.cc | 21 ++- storage/innobase/fsp/fsp0sysspace.cc | 17 +-- storage/innobase/include/buf0dblwr.h | 3 +- storage/innobase/include/log0recv.h | 18 ++- storage/innobase/log/log0recv.cc | 125 ++++++++++++---- storage/innobase/srv/srv0start.cc | 137 +++++++++--------- 12 files changed, 276 insertions(+), 136 deletions(-) diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index 7763aa9932218..ceb01ac480746 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -1,7 +1,7 @@ # # MDEV-32242 innodb.doublewrite test case always is skipped # -create table t1 (f1 int primary key, f2 blob) engine=innodb; +create table t1 (f1 int primary key, f2 blob) stats_persistent=0, engine=innodb; start transaction; insert into t1 values(1, repeat('#',12)); insert into t1 values(2, repeat('+',12)); @@ -19,6 +19,7 @@ XA PREPARE 'x'; disconnect dml; connection default; flush table t1 for export; +# Kill the server # restart FOUND 1 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err FOUND 1 /InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page number=3\]/ in mysqld.1.err @@ -33,5 +34,27 @@ f1 f2 3 //////////// 4 ------------ 5 ............ +connect dml,localhost,root,,; +XA START 'x'; +insert into t1 values (6, repeat('%', @@innodb_page_size/2)); +XA END 'x'; +XA PREPARE 'x'; +disconnect dml; +connection default; +flush table t1 for export; +# Kill the server +# restart +FOUND 2 /InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile/ in mysqld.1.err +XA ROLLBACK 'x'; +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select f1, f2 from t1; +f1 f2 +1 ############ +2 ++++++++++++ +3 //////////// +4 ------------ +5 ............ drop table t1; # End of 10.5 tests diff --git a/mysql-test/suite/innodb/r/log_file_name.result b/mysql-test/suite/innodb/r/log_file_name.result index 42b988ed3ca39..3d48a24d217de 100644 --- a/mysql-test/suite/innodb/r/log_file_name.result +++ b/mysql-test/suite/innodb/r/log_file_name.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); SET GLOBAL innodb_file_per_table=ON; FLUSH TABLES; CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; @@ -89,7 +90,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /\[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err +FOUND 1 /\[ERROR\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err FOUND 1 /\[ERROR\] InnoDB: Datafile .*u1.*\. Cannot determine the space ID from the first 64 pages/ in mysqld.1.err NOT FOUND /\[Note\] InnoDB: Cannot read first page of .*u2.ibd/ in mysqld.1.err # Fault 7: Missing or wrong data file and innodb_force_recovery @@ -98,11 +99,11 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /\[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err +FOUND 1 /\[ERROR\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err FOUND 1 /InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace/ in mysqld.1.err FOUND 1 /\[ERROR\] InnoDB: Cannot replay rename of tablespace \d+ from '.*u4.ibd' to '.*u6.ibd' because the target file exists/ in mysqld.1.err # restart: --innodb-force-recovery=1 -FOUND 1 /\[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err +FOUND 1 /\[ERROR\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err FOUND 1 /InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace/ in mysqld.1.err FOUND 1 /\[Warning\] InnoDB: Tablespace \d+ was not found at .*u[1-5].ibd, and innodb_force_recovery was set. All redo log for this tablespace will be ignored!/ in mysqld.1.err # restart diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index 89e6577b43405..541e4d23a1914 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -17,6 +17,7 @@ call mtr.add_suppression("InnoDB: A bad Space ID was found in datafile"); call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: "); call mtr.add_suppression("InnoDB: Inconsistent tablespace ID in .*t1\\.ibd"); call mtr.add_suppression("\\[Warning\\] Found 1 prepared XA transactions"); +call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -24,7 +25,7 @@ let MYSQLD_DATADIR=`select @@datadir`; let ALGO=`select @@innodb_checksum_algorithm`; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; -create table t1 (f1 int primary key, f2 blob) engine=innodb; +create table t1 (f1 int primary key, f2 blob) stats_persistent=0, engine=innodb; start transaction; insert into t1 values(1, repeat('#',12)); @@ -38,7 +39,7 @@ commit work; SET GLOBAL innodb_fast_shutdown = 0; let $shutdown_timeout=; --source include/restart_mysqld.inc - +--source ../include/no_checkpoint_start.inc connect (dml,localhost,root,,); XA START 'x'; insert into t1 values (6, repeat('%', @@innodb_page_size/2)); @@ -50,8 +51,8 @@ connection default; flush table t1 for export; let $restart_parameters=; -let $shutdown_timeout=0; ---source include/shutdown_mysqld.inc +--let CLEANUP_IF_CHECKPOINT=drop table t1, unexpected_checkpoint; +--source ../include/no_checkpoint_end.inc perl; use IO::Handle; @@ -119,6 +120,35 @@ let SEARCH_PATTERN=InnoDB: Recovered page \[page id: space=[1-9][0-9]*, page num XA ROLLBACK 'x'; check table t1; select f1, f2 from t1; + +--source ../include/no_checkpoint_start.inc +connect (dml,localhost,root,,); +XA START 'x'; +insert into t1 values (6, repeat('%', @@innodb_page_size/2)); +XA END 'x'; +XA PREPARE 'x'; +disconnect dml; +connection default; + +flush table t1 for export; + +let $restart_parameters=; +--source ../include/no_checkpoint_end.inc + +# Zero out the first page in file and try to recover from dblwr +perl; +use IO::Handle; +open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}test/t1.ibd") or die; +syswrite(FILE, chr(0) x $ENV{INNODB_PAGE_SIZE}); +close FILE; +EOF + +--source include/start_mysqld.inc +let SEARCH_PATTERN=InnoDB: Restoring page \[page id: space=[1-9][0-9]*, page number=0\] of datafile; +--source include/search_pattern_in_file.inc +XA ROLLBACK 'x'; +check table t1; +select f1, f2 from t1; drop table t1; --echo # End of 10.5 tests diff --git a/mysql-test/suite/innodb/t/doublewrite_debug.test b/mysql-test/suite/innodb/t/doublewrite_debug.test index 1bff8b4e07fe9..3d96a74e698fe 100644 --- a/mysql-test/suite/innodb/t/doublewrite_debug.test +++ b/mysql-test/suite/innodb/t/doublewrite_debug.test @@ -17,6 +17,7 @@ call mtr.add_suppression("Plugin 'InnoDB' (init function returned error|registra call mtr.add_suppression("InnoDB: A bad Space ID was found in datafile"); call mtr.add_suppression("InnoDB: Checksum mismatch in datafile: "); call mtr.add_suppression("InnoDB: Inconsistent tablespace ID in .*t1\\.ibd"); +call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; diff --git a/mysql-test/suite/innodb/t/log_file_name.test b/mysql-test/suite/innodb/t/log_file_name.test index 11eca0a7a1250..494d256cd1131 100644 --- a/mysql-test/suite/innodb/t/log_file_name.test +++ b/mysql-test/suite/innodb/t/log_file_name.test @@ -7,6 +7,8 @@ # Embedded server does not support crashing --source include/not_embedded.inc +call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile:"); + SET GLOBAL innodb_file_per_table=ON; FLUSH TABLES; @@ -172,6 +174,7 @@ call mtr.add_suppression("InnoDB: Table test/u[123] in the InnoDB data dictionar call mtr.add_suppression("InnoDB: Cannot replay rename of tablespace.*"); call mtr.add_suppression("InnoDB: Attempted to open a previously opened tablespace"); call mtr.add_suppression("InnoDB: Recovery cannot access file"); +call mtr.add_suppression("InnoDB: Cannot read first page in datafile:"); FLUSH TABLES; --enable_query_log @@ -215,7 +218,7 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; -let SEARCH_PATTERN= \[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; +let SEARCH_PATTERN= \[ERROR\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; --source include/search_pattern_in_file.inc let SEARCH_PATTERN= \[ERROR\] InnoDB: Datafile .*u1.*\. Cannot determine the space ID from the first 64 pages; @@ -240,7 +243,7 @@ let SEARCH_PATTERN= \[Note\] InnoDB: Cannot read first page of .*u2.ibd; --source include/start_mysqld.inc eval $check_no_innodb; -let SEARCH_PATTERN= \[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; +let SEARCH_PATTERN= \[ERROR\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; --source include/search_pattern_in_file.inc let SEARCH_PATTERN= InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace; @@ -253,7 +256,7 @@ let SEARCH_PATTERN= \[ERROR\] InnoDB: Cannot replay rename of tablespace \d+ fro --source include/restart_mysqld.inc -let SEARCH_PATTERN= \[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; +let SEARCH_PATTERN= \[ERROR\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; --source include/search_pattern_in_file.inc let SEARCH_PATTERN= InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 57ec3553b8dbe..d5c954dff89b6 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -324,11 +324,14 @@ dberr_t buf_dblwr_t::init_or_load_pages(pfs_os_file_t file, const char *path) os_file_flush(file); } else - for (ulint i= 0; i < size * 2; i++, page += srv_page_size) - if (mach_read_from_8(my_assume_aligned<8>(page + FIL_PAGE_LSN))) - /* Each valid page header must contain a nonzero FIL_PAGE_LSN field. */ + { + alignas(8) char checkpoint[8]; + mach_write_to_8(checkpoint, log_sys.next_checkpoint_lsn); + for (auto i= size * 2; i--; page += srv_page_size) + if (memcmp_aligned<8>(page + FIL_PAGE_LSN, checkpoint, 8) >= 0) + /* Valid pages are not older than the log checkpoint. */ recv_sys.dblwr.add(page); - + } err= DB_SUCCESS; goto func_exit; } diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index f0ead3b80a37a..653c848953a2f 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -475,9 +475,16 @@ Datafile::validate_for_recovery() err = find_space_id(); if (err != DB_SUCCESS || m_space_id == 0) { - ib::error() << "Datafile '" << m_filepath << "' is" - " corrupted. Cannot determine the space ID from" - " the first 64 pages."; + + m_space_id = recv_sys.dblwr.find_first_page( + m_filepath, m_handle); + + if (m_space_id) goto free_first_page; + + sql_print_error( + "InnoDB: Datafile '%s' is corrupted." + " Cannot determine the space ID from" + " the first 64 pages.", m_filepath); return(err); } @@ -485,7 +492,7 @@ Datafile::validate_for_recovery() m_space_id, m_filepath, m_handle)) { return(DB_CORRUPTION); } - +free_first_page: /* Free the previously read first page and then re-validate. */ free_first_page(); err = validate_first_page(0); @@ -531,9 +538,9 @@ Datafile::validate_first_page(lsn_t* flush_lsn) if (error_txt != NULL) { err_exit: - ib::info() << error_txt << " in datafile: " << m_filepath - << ", Space ID:" << m_space_id << ", Flags: " - << m_flags; + sql_print_error("InnoDB: %s in datafile: %s, Space ID: %zu, " + "Flags: %zu", error_txt, m_filepath, + m_space_id, m_flags); m_is_valid = false; free_first_page(); return(DB_CORRUPTION); diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index a7bb417c502f1..eb6e8d1166eb5 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -574,7 +574,7 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) } err = it->read_first_page( - m_ignore_read_only ? false : srv_read_only_mode); + m_ignore_read_only && srv_read_only_mode); if (err != DB_SUCCESS) { return(err); @@ -588,20 +588,17 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) /* Check the contents of the first page of the first datafile. */ - for (int retry = 0; retry < 2; ++retry) { + err = it->validate_first_page(flushed_lsn); - err = it->validate_first_page(flushed_lsn); - - if (err != DB_SUCCESS - && (retry == 1 - || recv_sys.dblwr.restore_first_page( + if (err != DB_SUCCESS) { + if (recv_sys.dblwr.restore_first_page( it->m_space_id, it->m_filepath, - it->handle()))) { - + it->handle())) { it->close(); - return(err); } + err = it->read_first_page( + m_ignore_read_only && srv_read_only_mode); } /* Make sure the tablespace space ID matches the diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index f82baeec89a40..99a41d069ef35 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -103,7 +103,8 @@ class buf_dblwr_t If we are upgrading from a version before MySQL 4.1, then this function performs the necessary update operations to support innodb_file_per_table. If we are in a crash recovery, this function - loads the pages from double write buffer into memory. + loads the pages from double write buffer which are not older than + the checkpoint into memory. @param file File handle @param path Path name of file @return DB_SUCCESS or error code */ diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 34211392ba399..aca9f994acbbd 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -49,6 +49,11 @@ recv_find_max_checkpoint(ulint* max_field) ATTRIBUTE_COLD void recv_recover_page(fil_space_t* space, buf_page_t* bpage) MY_ATTRIBUTE((nonnull)); +/** Read the latest checkpoint information from log file +and store it in log_sys.next_checkpoint and recv_sys.mlog_checkpoint_lsn +@return error code or DB_SUCCESS */ +dberr_t recv_recovery_read_max_checkpoint(); + /** Start recovering from a redo log checkpoint. @param[in] flush_lsn FIL_PAGE_FILE_FLUSH_LSN of first system tablespace page @@ -141,7 +146,18 @@ struct recv_dblwr_t @param file tablespace file handle @return whether the operation failed */ bool restore_first_page( - ulint space_id, const char *name, os_file_t file); + ulint space_id, const char *name, pfs_os_file_t file); + + /** Restore the first page of the given tablespace from + doublewrite buffer. + 1) Find the page which has page_no as 0 + 2) Read first 3 pages from tablespace file + 3) Compare the space_ids from the pages with page0 which + was retrieved from doublewrite buffer + @param name tablespace filepath + @param file tablespace file handle + @return space_id or 0 in case of error */ + uint32_t find_first_page(const char *name, pfs_os_file_t file); typedef std::deque > list; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 05120871b0ac0..52649419ec708 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3415,6 +3415,46 @@ recv_init_crash_recovery_spaces(bool rescan, bool& missing_tablespace) return DB_SUCCESS; } +dberr_t recv_recovery_read_max_checkpoint() +{ + ut_ad(srv_operation <= SRV_OPERATION_EXPORT_RESTORED || + srv_operation == SRV_OPERATION_RESTORE || + srv_operation == SRV_OPERATION_RESTORE_EXPORT); + ut_d(mysql_mutex_lock(&buf_pool.mutex)); + ut_ad(UT_LIST_GET_LEN(buf_pool.LRU) == 0); + ut_ad(UT_LIST_GET_LEN(buf_pool.unzip_LRU) == 0); + ut_d(mysql_mutex_unlock(&buf_pool.mutex)); + + if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) + { + sql_print_information("InnoDB: innodb_force_recovery=6 skips redo log apply"); + return DB_SUCCESS; + } + + mysql_mutex_lock(&log_sys.mutex); + ulint max_cp; + dberr_t err= recv_find_max_checkpoint(&max_cp); + + if (err != DB_SUCCESS) + recv_sys.recovered_lsn= log_sys.get_lsn(); + else + { + byte* buf= log_sys.checkpoint_buf; + err= log_sys.log.read(max_cp, {buf, OS_FILE_LOG_BLOCK_SIZE}); + if (err == DB_SUCCESS) + { + log_sys.next_checkpoint_no= + mach_read_from_8(buf + LOG_CHECKPOINT_NO); + log_sys.next_checkpoint_lsn= + mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + recv_sys.mlog_checkpoint_lsn= + mach_read_from_8(buf + LOG_CHECKPOINT_END_LSN); + } + } + mysql_mutex_unlock(&log_sys.mutex); + return err; +} + /** Start recovering from a redo log checkpoint. @param[in] flush_lsn FIL_PAGE_FILE_FLUSH_LSN of first system tablespace page @@ -3422,12 +3462,8 @@ of first system tablespace page dberr_t recv_recovery_from_checkpoint_start(lsn_t flush_lsn) { - ulint max_cp_field; - lsn_t checkpoint_lsn; bool rescan = false; - ib_uint64_t checkpoint_no; lsn_t contiguous_lsn; - byte* buf; dberr_t err = DB_SUCCESS; ut_ad(srv_operation <= SRV_OPERATION_EXPORT_RESTORED @@ -3445,27 +3481,11 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) return(DB_SUCCESS); } - recv_sys.recovery_on = true; - mysql_mutex_lock(&log_sys.mutex); - - err = recv_find_max_checkpoint(&max_cp_field); - - if (err != DB_SUCCESS) { - - recv_sys.recovered_lsn = log_sys.get_lsn(); - mysql_mutex_unlock(&log_sys.mutex); - return(err); - } - - buf = log_sys.checkpoint_buf; - if ((err= log_sys.log.read(max_cp_field, {buf, OS_FILE_LOG_BLOCK_SIZE}))) { - mysql_mutex_unlock(&log_sys.mutex); - return(err); - } - - checkpoint_lsn = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); - checkpoint_no = mach_read_from_8(buf + LOG_CHECKPOINT_NO); + uint64_t checkpoint_no= log_sys.next_checkpoint_no; + lsn_t checkpoint_lsn= log_sys.next_checkpoint_lsn; + const lsn_t end_lsn= recv_sys.mlog_checkpoint_lsn; + recv_sys.recovery_on = true; /* Start reading the log from the checkpoint lsn. The variable contiguous_lsn contains an lsn up to which the log is known to @@ -3474,8 +3494,6 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) ut_ad(RECV_SCAN_SIZE <= srv_log_buffer_size); - const lsn_t end_lsn = mach_read_from_8( - buf + LOG_CHECKPOINT_END_LSN); ut_ad(recv_sys.pages.empty()); contiguous_lsn = checkpoint_lsn; @@ -3845,8 +3863,52 @@ byte *recv_dblwr_t::find_page(const page_id_t page_id, return result; } +uint32_t recv_dblwr_t::find_first_page(const char *name, pfs_os_file_t file) +{ + os_offset_t file_size= os_file_get_size(file); + if (file_size != (os_offset_t) -1) + { + for (const page_t *page : pages) + { + uint32_t space_id= page_get_space_id(page); + if (page_get_page_no(page) > 0 || space_id == 0) +next_page: + continue; + uint32_t flags= mach_read_from_4( + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + page_id_t page_id(space_id, 0); + size_t page_size= fil_space_t::physical_size(flags); + if (file_size < 4 * page_size) + goto next_page; + byte *read_page= + static_cast(aligned_malloc(3 * page_size, page_size)); + /* Read 3 pages from the file and match the space id + with the space id which is stored in + doublewrite buffer page. */ + if (os_file_read(IORequestRead, file, read_page, page_size, + 3 * page_size) != DB_SUCCESS) + goto next_page; + for (ulint j= 0; j <= 2; j++) + { + byte *cur_page= read_page + j * page_size; + if (buf_is_zeroes(span(cur_page, page_size))) + return 0; + if (mach_read_from_4(cur_page + FIL_PAGE_OFFSET) != j + 1 || + memcmp(cur_page + FIL_PAGE_SPACE_ID, + page + FIL_PAGE_SPACE_ID, 4) || + buf_page_is_corrupted(false, cur_page, flags)) + goto next_page; + } + if (!restore_first_page(space_id, name, file)) + return space_id; + break; + } + } + return 0; +} + bool recv_dblwr_t::restore_first_page(ulint space_id, const char *name, - os_file_t file) + pfs_os_file_t file) { const page_id_t page_id(space_id, 0); const byte* page= find_page(page_id); @@ -3854,10 +3916,11 @@ bool recv_dblwr_t::restore_first_page(ulint space_id, const char *name, { /* If the first page of the given user tablespace is not there in the doublewrite buffer, then the recovery is going to fail - now. Hence this is treated as error. */ - ib::error() - << "Corrupted page " << page_id << " of datafile '" - << name <<"' could not be found in the doublewrite buffer."; + now. Report error only when doublewrite buffer is not empty */ + if (pages.size()) + ib::error() << "Corrupted page " << page_id << " of datafile '" + << name <<"' could not be found in the " + <<"doublewrite buffer."; return true; } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 56e41616d1de5..0760b8ca5fd67 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -297,9 +297,6 @@ static dberr_t create_log_file(bool create_new_db, lsn_t lsn, log_sys.log.create(); log_sys.log.open_file(logfile0); - if (!fil_system.sys_space->open(create_new_db)) { - return DB_ERROR; - } /* Create a log checkpoint. */ mysql_mutex_lock(&log_sys.mutex); @@ -1274,40 +1271,6 @@ dberr_t srv_start(bool create_new_db) /* Check if undo tablespaces and redo log files exist before creating a new system tablespace */ - if (create_new_db) { - err = srv_check_undo_redo_logs_exists(); - if (err != DB_SUCCESS) { - return(srv_init_abort(DB_ERROR)); - } - recv_sys.debug_free(); - } - - /* Open or create the data files. */ - ulint sum_of_new_sizes; - - err = srv_sys_space.open_or_create( - false, create_new_db, &sum_of_new_sizes, &flushed_lsn); - - switch (err) { - case DB_SUCCESS: - break; - case DB_CANNOT_OPEN_FILE: - ib::error() - << "Could not open or create the system tablespace. If" - " you tried to add new data files to the system" - " tablespace, and it failed here, you should now" - " edit innodb_data_file_path in my.cnf back to what" - " it was, and remove the new ibdata files InnoDB" - " created in this failed attempt. InnoDB only wrote" - " those files full of zeros, but did not yet use" - " them in any way. But be careful: do not remove" - " old data files which contain your precious data!"; - /* fall through */ - default: - /* Other errors might come from Datafile::validate_first_page() */ - return(srv_init_abort(err)); - } - srv_log_file_size_requested = srv_log_file_size; if (innodb_encrypt_temporary_tables && !log_crypt_init()) { @@ -1317,18 +1280,17 @@ dberr_t srv_start(bool create_new_db) std::string logfile0; bool create_new_log = create_new_db; if (create_new_db) { + err = srv_check_undo_redo_logs_exists(); + if (err != DB_SUCCESS) { + return(srv_init_abort(DB_ERROR)); + } + recv_sys.debug_free(); flushed_lsn = log_sys.get_lsn(); log_sys.set_flushed_lsn(flushed_lsn); err = create_log_file(true, flushed_lsn, logfile0); if (err != DB_SUCCESS) { - for (Tablespace::const_iterator - i = srv_sys_space.begin(); - i != srv_sys_space.end(); i++) { - os_file_delete(innodb_data_file_key, - i->filepath()); - } return(srv_init_abort(err)); } } else { @@ -1343,51 +1305,84 @@ dberr_t srv_start(bool create_new_db) } create_new_log = srv_log_file_size == 0; - if (create_new_log) { - if (flushed_lsn < lsn_t(1000)) { - ib::error() - << "Cannot create log file because" - " data files are corrupt or the" - " database was not shut down cleanly" - " after creating the data files."; - return srv_init_abort(DB_ERROR); - } + if (!create_new_log) { + srv_log_file_found = log_file_found; - srv_log_file_size = srv_log_file_size_requested; + log_sys.log.open_file(get_log_file_path()); - err = create_log_file(false, flushed_lsn, logfile0); + log_sys.log.create(); - if (err == DB_SUCCESS) { - err = create_log_file_rename(flushed_lsn, - logfile0); + if (!log_set_capacity( + srv_log_file_size_requested)) { + return(srv_init_abort(DB_ERROR)); } + /* Enable checkpoints in the page cleaner. */ + recv_sys.recovery_on = false; + + err= recv_recovery_read_max_checkpoint(); + if (err != DB_SUCCESS) { - return(srv_init_abort(err)); + return srv_init_abort(err); } - - /* Suppress the message about - crash recovery. */ - flushed_lsn = log_sys.get_lsn(); - goto file_checked; } + } + + /* Open or create the data files. */ + ulint sum_of_new_sizes; + + err = srv_sys_space.open_or_create( + false, create_new_db, &sum_of_new_sizes, &flushed_lsn); + + switch (err) { + case DB_SUCCESS: + break; + case DB_CANNOT_OPEN_FILE: + sql_print_error("InnoDB: Could not open or create the system" + " tablespace. If you tried to add new data files" + " to the system tablespace, and it failed here," + " you should now edit innodb_data_file_path" + " in my.cnf back to what it was, and remove the" + " new ibdata files InnoDB created in this failed" + " attempt. InnoDB only wrote those files full of" + " zeros, but did not yet use them in any way. But" + " be careful: do not remove old data files which" + " contain your precious data!"); + /* fall through */ + default: + /* Other errors might come from Datafile::validate_first_page() */ + return(srv_init_abort(err)); + } - srv_log_file_found = log_file_found; + if (!create_new_db && create_new_log) { + if (flushed_lsn < lsn_t(1000)) { + sql_print_error( + "InnoDB: Cannot create log file because" + " data files are corrupt or the" + " database was not shut down cleanly" + " after creating the data files."); + return srv_init_abort(DB_ERROR); + } - log_sys.log.open_file(get_log_file_path()); + srv_log_file_size = srv_log_file_size_requested; - log_sys.log.create(); + err = create_log_file(false, flushed_lsn, logfile0); + if (err == DB_SUCCESS) { + err = create_log_file_rename(flushed_lsn, logfile0); + } - if (!log_set_capacity(srv_log_file_size_requested)) { - return(srv_init_abort(DB_ERROR)); + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); } - /* Enable checkpoints in the page cleaner. */ - recv_sys.recovery_on = false; + /* Suppress the message about + crash recovery. */ + flushed_lsn = log_sys.get_lsn(); + goto file_checked; } file_checked: - /* Open log file and data files in the systemtablespace: we keep + /* Open data files in the systemtablespace: we keep them open until database shutdown */ ut_d(fil_system.sys_space->recv_size = srv_sys_space_size_debug); From 931df937e9382248ab082e4b3abd8ed149d48cd4 Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 16 Jan 2024 17:17:50 +1100 Subject: [PATCH 28/33] MDEV-32559 failing spider signal_ddl_recovery_done callback should result in spider deinit Since 0930eb86cb00edb2ae285c3250ca51bae126e5e5, system table creation needed for spider init is delayed to the signal_ddl_recovery_done callback. Since it is part of the init, failure should result in spider deinit. We also remove the call to spider_init_system_tables() from spider_db_init(), as it was removed in the commit mentioned above and accidentally restored in a merge. --- include/mysql/plugin.h | 16 ++++++++++++++-- sql/handler.cc | 3 ++- sql/handler.h | 2 +- sql/mysqld.cc | 8 ++++++++ storage/innobase/handler/ha_innodb.cc | 3 ++- .../spider/bugfix/r/signal_ddl_fail.result | 8 ++++++++ .../spider/bugfix/t/signal_ddl_fail.opt | 2 ++ .../spider/bugfix/t/signal_ddl_fail.test | 10 ++++++++++ storage/spider/spd_table.cc | 19 +++---------------- 9 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt create mode 100644 storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index 89ecf2b34c1c7..8a7a5e142231e 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -531,7 +531,13 @@ struct st_mysql_plugin const char *author; /* plugin author (for I_S.PLUGINS) */ const char *descr; /* general descriptive text (for I_S.PLUGINS) */ int license; /* the plugin license (PLUGIN_LICENSE_XXX) */ - int (*init)(void *); /* the function to invoke when plugin is loaded */ + /* + The function to invoke when plugin is loaded. Plugin + initialisation done here should defer any ALTER TABLE queries to + after the ddl recovery is done, in the signal_ddl_recovery_done() + callback called by ha_signal_ddl_recovery_done(). + */ + int (*init)(void *); int (*deinit)(void *);/* the function to invoke when plugin is unloaded */ unsigned int version; /* plugin version (for I_S.PLUGINS) */ struct st_mysql_show_var *status_vars; @@ -555,7 +561,13 @@ struct st_maria_plugin const char *author; /* plugin author (for SHOW PLUGINS) */ const char *descr; /* general descriptive text (for SHOW PLUGINS ) */ int license; /* the plugin license (PLUGIN_LICENSE_XXX) */ - int (*init)(void *); /* the function to invoke when plugin is loaded */ + /* + The function to invoke when plugin is loaded. Plugin + initialisation done here should defer any ALTER TABLE queries to + after the ddl recovery is done, in the signal_ddl_recovery_done() + callback called by ha_signal_ddl_recovery_done(). + */ + int (*init)(void *); int (*deinit)(void *);/* the function to invoke when plugin is unloaded */ unsigned int version; /* plugin version (for SHOW PLUGINS) */ struct st_mysql_show_var *status_vars; diff --git a/sql/handler.cc b/sql/handler.cc index 58a2eb3e43acd..86fc697ec0c22 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -968,7 +968,8 @@ static my_bool signal_ddl_recovery_done(THD *, plugin_ref plugin, void *) { handlerton *hton= plugin_hton(plugin); if (hton->signal_ddl_recovery_done) - (hton->signal_ddl_recovery_done)(hton); + if ((hton->signal_ddl_recovery_done)(hton)) + plugin_ref_to_int(plugin)->state= PLUGIN_IS_DELETED; return 0; } diff --git a/sql/handler.h b/sql/handler.h index 0296d30213fa2..2d01979ab481f 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -1565,7 +1565,7 @@ struct handlerton const LEX_CUSTRING *version, ulonglong create_id); /* Called for all storage handlers after ddl recovery is done */ - void (*signal_ddl_recovery_done)(handlerton *hton); + int (*signal_ddl_recovery_done)(handlerton *hton); /* Optional clauses in the CREATE/ALTER TABLE diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 952802541b954..8b7888dfb79a2 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5118,6 +5118,14 @@ static int init_server_components() tc_log= 0; // ha_initialize_handlerton() needs that + /* + Plugins may not be completed because system table DDLs are only + run after the ddl recovery done. Therefore between the + plugin_init() call and the ha_signal_ddl_recovery_done() call + below only things related to preparation for recovery should be + done and nothing else, and definitely not anything assuming that + all plugins have been initialised. + */ if (plugin_init(&remaining_argc, remaining_argv, (opt_noacl ? PLUGIN_INIT_SKIP_PLUGIN_TABLE : 0) | (opt_abort ? PLUGIN_INIT_SKIP_INITIALIZATION : 0))) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e3b18599c99ec..af556d656951b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2155,7 +2155,7 @@ static void drop_garbage_tables_after_restore() ut_d(purge_sys.resume_FTS()); } -static void innodb_ddl_recovery_done(handlerton*) +static int innodb_ddl_recovery_done(handlerton*) { ut_ad(!ddl_recovery_done); ut_d(ddl_recovery_done= true); @@ -2166,6 +2166,7 @@ static void innodb_ddl_recovery_done(handlerton*) drop_garbage_tables_after_restore(); srv_init_purge_tasks(); } + return 0; } /********************************************************************//** diff --git a/storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result b/storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result new file mode 100644 index 0000000000000..c86e600b2c1b2 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/signal_ddl_fail.result @@ -0,0 +1,8 @@ +# +# MDEV-32559 Move alter table statements in spider init queries to be executed in the signal_ddl_recovery_done callback +# +select * from mysql.plugin; +name dl +# +# end of test signal_ddl_fail +# diff --git a/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt new file mode 100644 index 0000000000000..d883df7bed296 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.opt @@ -0,0 +1,2 @@ +--plugin-load-add=ha_spider +--debug-dbug=d,fail_spider_ddl_recovery_done diff --git a/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test new file mode 100644 index 0000000000000..f13eae3a06293 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/signal_ddl_fail.test @@ -0,0 +1,10 @@ +--source include/have_debug.inc +--echo # +--echo # MDEV-32559 Move alter table statements in spider init queries to be executed in the signal_ddl_recovery_done callback +--echo # +# This test tests that failure in ddl_recovery callback causes the +# plugin to be deinitialized. +select * from mysql.plugin; +--echo # +--echo # end of test signal_ddl_fail +--echo # diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index 65a00f5da3476..5dc489daca33c 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -7127,9 +7127,10 @@ bool spider_init_system_tables() Spider is typically loaded before ddl_recovery, but DDL statements cannot be executed before ddl_recovery, so we delay system table creation. */ -static void spider_after_ddl_recovery(handlerton *) +static int spider_after_ddl_recovery(handlerton *) { - spider_init_system_tables(); + DBUG_EXECUTE_IF("fail_spider_ddl_recovery_done", return 1;); + return spider_init_system_tables(); } int spider_db_init( @@ -7151,16 +7152,6 @@ int spider_db_init( #ifdef HTON_CAN_READ_CONNECT_STRING_IN_PARTITION spider_hton->flags |= HTON_CAN_READ_CONNECT_STRING_IN_PARTITION; #endif - /* spider_hton->db_type = DB_TYPE_SPIDER; */ - /* - spider_hton->savepoint_offset; - spider_hton->savepoint_set = spider_savepoint_set; - spider_hton->savepoint_rollback = spider_savepoint_rollback; - spider_hton->savepoint_release = spider_savepoint_release; - spider_hton->create_cursor_read_view = spider_create_cursor_read_view; - spider_hton->set_cursor_read_view = spider_set_cursor_read_view; - spider_hton->close_cursor_read_view = spider_close_cursor_read_view; - */ spider_hton->panic = spider_panic; spider_hton->signal_ddl_recovery_done= spider_after_ddl_recovery; spider_hton->close_connection = spider_close_connection; @@ -7233,10 +7224,6 @@ int spider_db_init( #ifndef WITHOUT_SPIDER_BG_SEARCH if (pthread_attr_init(&spider_pt_attr)) goto error_pt_attr_init; -/* - if (pthread_attr_setdetachstate(&spider_pt_attr, PTHREAD_CREATE_DETACHED)) - goto error_pt_attr_setstate; -*/ #endif #if MYSQL_VERSION_ID < 50500 From fa3171df08dc9b274dfe7fa03205b66554b621ac Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 27 Dec 2023 18:57:49 +0400 Subject: [PATCH 29/33] MDEV-27666 User variable not parsed as geometry variable in geometry function Adding GEOMETRY type user variables. --- mysql-test/main/gis.result | 107 ++++++++++++++++++ mysql-test/main/gis.test | 63 +++++++++++ .../binlog/r/binlog_gis_user_var_stm.result | 12 ++ .../binlog/t/binlog_gis_user_var_stm.test | 15 +++ .../suite/rpl/r/rpl_gis_user_var.result | 21 ++++ mysql-test/suite/rpl/t/rpl_gis_user_var.test | 18 +++ plugin/user_variables/user_variables.cc | 6 +- sql/item_func.cc | 80 ++++++------- sql/item_func.h | 7 +- sql/log.cc | 10 +- sql/log_event.cc | 54 ++++++--- sql/log_event.h | 21 ++-- sql/log_event_client.cc | 7 +- sql/log_event_data_type.h | 74 ++++++++++++ sql/log_event_server.cc | 68 +++++++---- sql/sql_binlog.cc | 5 +- sql/sql_class.h | 7 +- sql/sql_repl.cc | 2 +- sql/sql_type.cc | 28 +++++ sql/sql_type.h | 11 ++ sql/sql_type_geom.h | 7 ++ storage/spider/spd_conn.cc | 2 +- storage/spider/spd_table.cc | 2 +- 23 files changed, 509 insertions(+), 118 deletions(-) create mode 100644 mysql-test/suite/binlog/r/binlog_gis_user_var_stm.result create mode 100644 mysql-test/suite/binlog/t/binlog_gis_user_var_stm.test create mode 100644 mysql-test/suite/rpl/r/rpl_gis_user_var.result create mode 100644 mysql-test/suite/rpl/t/rpl_gis_user_var.test create mode 100644 sql/log_event_data_type.h diff --git a/mysql-test/main/gis.result b/mysql-test/main/gis.result index f8816090b0488..258d0c2f050e3 100644 --- a/mysql-test/main/gis.result +++ b/mysql-test/main/gis.result @@ -5328,5 +5328,112 @@ SELECT BIT_XOR(a) FROM t1; ERROR HY000: Illegal parameter data type geometry for operation 'bit_xor(' DROP TABLE t1; # +# MDEV-27666 User variable not parsed as geometry variable in geometry function. +# +set @g= point(1, 1); +select ST_AsWKT(GeometryCollection(Point(44, 6), @g)); +ST_AsWKT(GeometryCollection(Point(44, 6), @g)) +GEOMETRYCOLLECTION(POINT(44 6),POINT(1 1)) +set @g= "just a string"; +select ST_AsWKT(GeometryCollection(Point(44, 6), @g)); +ERROR HY000: Illegal parameter data type longblob for operation 'geometrycollection' +SET @g= LineString(Point(0,0), Point(0,1)); +SELECT AsText(PointN(@g, 1)); +AsText(PointN(@g, 1)) +POINT(0 0) +SELECT AsText(PointN(@g, 2)); +AsText(PointN(@g, 2)) +POINT(0 1) +SET @g= Point(1, 1); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` point DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +POINT(1 1) +DROP TABLE t1; +SET @g= MultiPoint(Point(1, 1), Point(-1,-1)); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` multipoint DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +MULTIPOINT(1 1,-1 -1) +DROP TABLE t1; +SET @g= LineString(Point(1, 1), Point(2,2)); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` linestring DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +LINESTRING(1 1,2 2) +DROP TABLE t1; +SET @g= MultiLineString(LineString(Point(1, 1), Point(2,2)), +LineString(Point(-1, -1), Point(-2,-2))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` multilinestring DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +MULTILINESTRING((1 1,2 2),(-1 -1,-2 -2)) +DROP TABLE t1; +SET @g= Polygon(LineString(Point(0, 0), Point(30, 0), Point(30, 30), Point(0, 0))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` polygon DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +POLYGON((0 0,30 0,30 30,0 0)) +DROP TABLE t1; +SET @g= MultiPolygon(Polygon(LineString(Point(0, 3), Point(3, 3), +Point(3, 0), Point(0, 3)))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` multipolygon DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +MULTIPOLYGON(((0 3,3 3,3 0,0 3))) +DROP TABLE t1; +SET @g= GeometryCollection(Point(44, 6), LineString(Point(3, 6), Point(7, 9))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` geometrycollection DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +GEOMETRYCOLLECTION(POINT(44 6),LINESTRING(3 6,7 9)) +DROP TABLE t1; +SET @g= GeometryFromText('POINT(1 1)'); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `g` geometry DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT AsText(g) FROM t1; +AsText(g) +POINT(1 1) +DROP TABLE t1; +# # End of 10.5 tests # diff --git a/mysql-test/main/gis.test b/mysql-test/main/gis.test index 4d66fb2b07d95..f160d8f1a1067 100644 --- a/mysql-test/main/gis.test +++ b/mysql-test/main/gis.test @@ -3374,6 +3374,69 @@ SELECT BIT_OR(a) FROM t1; SELECT BIT_XOR(a) FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-27666 User variable not parsed as geometry variable in geometry function. +--echo # + +set @g= point(1, 1); +select ST_AsWKT(GeometryCollection(Point(44, 6), @g)); +set @g= "just a string"; +--error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION +select ST_AsWKT(GeometryCollection(Point(44, 6), @g)); + +SET @g= LineString(Point(0,0), Point(0,1)); +SELECT AsText(PointN(@g, 1)); +SELECT AsText(PointN(@g, 2)); + +SET @g= Point(1, 1); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= MultiPoint(Point(1, 1), Point(-1,-1)); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= LineString(Point(1, 1), Point(2,2)); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= MultiLineString(LineString(Point(1, 1), Point(2,2)), + LineString(Point(-1, -1), Point(-2,-2))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= Polygon(LineString(Point(0, 0), Point(30, 0), Point(30, 30), Point(0, 0))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= MultiPolygon(Polygon(LineString(Point(0, 3), Point(3, 3), + Point(3, 0), Point(0, 3)))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= GeometryCollection(Point(44, 6), LineString(Point(3, 6), Point(7, 9))); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; + +SET @g= GeometryFromText('POINT(1 1)'); +CREATE TABLE t1 AS SELECT @g AS g; +SHOW CREATE TABLE t1; +SELECT AsText(g) FROM t1; +DROP TABLE t1; --echo # --echo # End of 10.5 tests diff --git a/mysql-test/suite/binlog/r/binlog_gis_user_var_stm.result b/mysql-test/suite/binlog/r/binlog_gis_user_var_stm.result new file mode 100644 index 0000000000000..e467c9c89b457 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_gis_user_var_stm.result @@ -0,0 +1,12 @@ +SET @g0= POINT(1,1); +SET @g1= Polygon(LineString(Point(0, 0), Point(30, 0), Point(30, 30), Point(0, 0))); +CREATE TABLE t1 AS SELECT @g0 AS g0, @g1 AS g1; +DROP TABLE t1; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # User var # # @`g0`=/*point*/_binary X'000000000101000000000000000000F03F000000000000F03F' COLLATE binary +master-bin.000001 # User var # # @`g1`=/*polygon*/_binary X'0000000001030000000100000004000000000000000000000000000000000000000000000000003E4000000000000000000000000000003E400000000000003E4000000000000000000000000000000000' COLLATE binary +master-bin.000001 # Query # # use `test`; CREATE TABLE t1 AS SELECT @g0 AS g0, @g1 AS g1 +master-bin.000001 # Gtid # # GTID #-#-# +master-bin.000001 # Query # # use `test`; DROP TABLE `t1` /* generated by server */ diff --git a/mysql-test/suite/binlog/t/binlog_gis_user_var_stm.test b/mysql-test/suite/binlog/t/binlog_gis_user_var_stm.test new file mode 100644 index 0000000000000..7e789cd7ae44d --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_gis_user_var_stm.test @@ -0,0 +1,15 @@ +--source include/not_embedded.inc +--source include/have_binlog_format_statement.inc +--source include/have_geometry.inc + +--disable_query_log +reset master; # get rid of previous tests binlog +--enable_query_log + +SET @g0= POINT(1,1); +SET @g1= Polygon(LineString(Point(0, 0), Point(30, 0), Point(30, 30), Point(0, 0))); +CREATE TABLE t1 AS SELECT @g0 AS g0, @g1 AS g1; +DROP TABLE t1; + +--let $binlog_file = LAST +source include/show_binlog_events.inc; diff --git a/mysql-test/suite/rpl/r/rpl_gis_user_var.result b/mysql-test/suite/rpl/r/rpl_gis_user_var.result new file mode 100644 index 0000000000000..c6aab9e03f0f3 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_gis_user_var.result @@ -0,0 +1,21 @@ +include/master-slave.inc +[connection master] +# +# +# +connection master; +SET @p=POINT(1,1); +CREATE TABLE t1 AS SELECT @p AS p; +connection slave; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `p` point DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +SELECT ST_AsWKT(p) FROM t1; +ST_AsWKT(p) +POINT(1 1) +connection master; +DROP TABLE t1; +connection slave; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_gis_user_var.test b/mysql-test/suite/rpl/t/rpl_gis_user_var.test new file mode 100644 index 0000000000000..8edd8cb930939 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_gis_user_var.test @@ -0,0 +1,18 @@ +--source include/have_geometry.inc +--source include/master-slave.inc + +--echo # +--echo # +--echo # + +connection master; +SET @p=POINT(1,1); +CREATE TABLE t1 AS SELECT @p AS p; +sync_slave_with_master; +SHOW CREATE TABLE t1; +SELECT ST_AsWKT(p) FROM t1; +connection master; +DROP TABLE t1; +sync_slave_with_master; + +--source include/rpl_end.inc diff --git a/plugin/user_variables/user_variables.cc b/plugin/user_variables/user_variables.cc index f820e4ad8900b..c76337cb084b8 100644 --- a/plugin/user_variables/user_variables.cc +++ b/plugin/user_variables/user_variables.cc @@ -79,9 +79,9 @@ static int user_variables_fill(THD *thd, TABLE_LIST *tables, COND *cond) else return 1; - const LEX_CSTRING *tmp= var->unsigned_flag ? - &unsigned_result_types[var->type] : - &result_types[var->type]; + const LEX_CSTRING *tmp= var->type_handler()->is_unsigned() ? + &unsigned_result_types[var->type_handler()->result_type()] : + &result_types[var->type_handler()->result_type()]; field[2]->store(tmp->str, tmp->length, system_charset_info); if (var->charset()) diff --git a/sql/item_func.cc b/sql/item_func.cc index 5202d395580a3..b2f2d6550f121 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4658,7 +4658,6 @@ user_var_entry *get_variable(HASH *hash, LEX_CSTRING *name, entry->length=0; entry->update_query_id=0; entry->set_charset(NULL); - entry->unsigned_flag= 0; /* If we are here, we were called from a SET or a query which sets a variable. Imagine it is this: @@ -4670,7 +4669,7 @@ user_var_entry *get_variable(HASH *hash, LEX_CSTRING *name, by Item_func_get_user_var (because that's not necessary). */ entry->used_query_id=current_thd->query_id; - entry->type=STRING_RESULT; + entry->set_handler(&type_handler_long_blob); memcpy((char*) entry->name.str, name->str, name->length+1); if (my_hash_insert(hash,(uchar*) entry)) { @@ -4746,9 +4745,12 @@ bool Item_func_set_user_var::fix_fields(THD *thd, Item **ref) switch (args[0]->result_type()) { case STRING_RESULT: case TIME_RESULT: - set_handler(type_handler_long_blob. - type_handler_adjusted_to_max_octet_length(max_length, - collation.collation)); + if (args[0]->field_type() == MYSQL_TYPE_GEOMETRY) + set_handler(args[0]->type_handler()); + else + set_handler(type_handler_long_blob. + type_handler_adjusted_to_max_octet_length(max_length, + collation.collation)); break; case REAL_RESULT: set_handler(&type_handler_double); @@ -4873,9 +4875,9 @@ bool Item_func_set_user_var::register_field_in_bitmap(void *arg) bool update_hash(user_var_entry *entry, bool set_null, void *ptr, size_t length, - Item_result type, CHARSET_INFO *cs, - bool unsigned_arg) + const Type_handler *th, CHARSET_INFO *cs) { + entry->set_handler(th); if (set_null) { char *pos= (char*) entry+ ALIGN_SIZE(sizeof(user_var_entry)); @@ -4886,7 +4888,7 @@ update_hash(user_var_entry *entry, bool set_null, void *ptr, size_t length, } else { - if (type == STRING_RESULT) + if (th->result_type() == STRING_RESULT) length++; // Store strings with end \0 if (length <= extra_size) { @@ -4915,20 +4917,18 @@ update_hash(user_var_entry *entry, bool set_null, void *ptr, size_t length, return 1; } } - if (type == STRING_RESULT) + if (th->result_type() == STRING_RESULT) { length--; // Fix length change above entry->value[length]= 0; // Store end \0 } if (length) memmove(entry->value, ptr, length); - if (type == DECIMAL_RESULT) + if (th->result_type() == DECIMAL_RESULT) ((my_decimal*)entry->value)->fix_buffer_pointer(); entry->length= length; entry->set_charset(cs); - entry->unsigned_flag= unsigned_arg; } - entry->type=type; #ifdef USER_VAR_TRACKING #ifndef EMBEDDED_LIBRARY THD *thd= current_thd; @@ -4941,9 +4941,8 @@ update_hash(user_var_entry *entry, bool set_null, void *ptr, size_t length, bool Item_func_set_user_var::update_hash(void *ptr, size_t length, - Item_result res_type, - CHARSET_INFO *cs, - bool unsigned_arg) + const Type_handler *th, + CHARSET_INFO *cs) { /* If we set a variable explicitly to NULL then keep the old @@ -4957,9 +4956,8 @@ Item_func_set_user_var::update_hash(void *ptr, size_t length, else null_value= args[0]->null_value; if (null_value && null_item) - res_type= m_var_entry->type; // Don't change type of item - if (::update_hash(m_var_entry, null_value, - ptr, length, res_type, cs, unsigned_arg)) + th= m_var_entry->type_handler(); // Don't change type of item + if (::update_hash(m_var_entry, null_value, ptr, length, th, cs)) { null_value= 1; return 1; @@ -4975,7 +4973,7 @@ double user_var_entry::val_real(bool *null_value) if ((*null_value= (value == 0))) return 0.0; - switch (type) { + switch (type_handler()->result_type()) { case REAL_RESULT: return *(double*) value; case INT_RESULT: @@ -5000,7 +4998,7 @@ longlong user_var_entry::val_int(bool *null_value) const if ((*null_value= (value == 0))) return 0; - switch (type) { + switch (type_handler()->result_type()) { case REAL_RESULT: return (longlong) *(double*) value; case INT_RESULT: @@ -5029,12 +5027,12 @@ String *user_var_entry::val_str(bool *null_value, String *str, if ((*null_value= (value == 0))) return (String*) 0; - switch (type) { + switch (type_handler()->result_type()) { case REAL_RESULT: str->set_real(*(double*) value, decimals, charset()); break; case INT_RESULT: - if (!unsigned_flag) + if (!type_handler()->is_unsigned()) str->set(*(longlong*) value, charset()); else str->set(*(ulonglong*) value, charset()); @@ -5061,7 +5059,7 @@ my_decimal *user_var_entry::val_decimal(bool *null_value, my_decimal *val) if ((*null_value= (value == 0))) return 0; - switch (type) { + switch (type_handler()->result_type()) { case REAL_RESULT: double2my_decimal(E_DEC_FATAL_ERROR, *(double*) value, val); break; @@ -5200,33 +5198,37 @@ Item_func_set_user_var::update() case REAL_RESULT: { res= update_hash((void*) &save_result.vreal,sizeof(save_result.vreal), - REAL_RESULT, &my_charset_numeric, 0); + &type_handler_double, &my_charset_numeric); break; } case INT_RESULT: { res= update_hash((void*) &save_result.vint, sizeof(save_result.vint), - INT_RESULT, &my_charset_numeric, unsigned_flag); + unsigned_flag ? (Type_handler *) &type_handler_ulonglong : + (Type_handler *) &type_handler_slonglong, + &my_charset_numeric); break; } case STRING_RESULT: { if (!save_result.vstr) // Null value - res= update_hash((void*) 0, 0, STRING_RESULT, &my_charset_bin, 0); + res= update_hash((void*) 0, 0, &type_handler_long_blob, &my_charset_bin); else res= update_hash((void*) save_result.vstr->ptr(), - save_result.vstr->length(), STRING_RESULT, - save_result.vstr->charset(), 0); + save_result.vstr->length(), + field_type() == MYSQL_TYPE_GEOMETRY ? + type_handler() : &type_handler_long_blob, + save_result.vstr->charset()); break; } case DECIMAL_RESULT: { if (!save_result.vdec) // Null value - res= update_hash((void*) 0, 0, DECIMAL_RESULT, &my_charset_bin, 0); + res= update_hash((void*) 0, 0, &type_handler_newdecimal, &my_charset_bin); else res= update_hash((void*) save_result.vdec, - sizeof(my_decimal), DECIMAL_RESULT, - &my_charset_numeric, 0); + sizeof(my_decimal), &type_handler_newdecimal, + &my_charset_numeric); break; } case ROW_RESULT: @@ -5618,9 +5620,8 @@ get_var_with_binlog(THD *thd, enum_sql_command sql_command, user_var_event->value= (char*) user_var_event + ALIGN_SIZE(sizeof(BINLOG_USER_VAR_EVENT)); user_var_event->user_var_event= var_entry; - user_var_event->type= var_entry->type; + user_var_event->th= var_entry->type_handler(); user_var_event->charset_number= var_entry->charset()->number; - user_var_event->unsigned_flag= var_entry->unsigned_flag; if (!var_entry->value) { /* NULL value*/ @@ -5663,9 +5664,9 @@ bool Item_func_get_user_var::fix_length_and_dec() */ if (likely(!error && m_var_entry)) { - unsigned_flag= m_var_entry->unsigned_flag; + unsigned_flag= m_var_entry->type_handler()->is_unsigned(); max_length= (uint32)m_var_entry->length; - switch (m_var_entry->type) { + switch (m_var_entry->type_handler()->result_type()) { case REAL_RESULT: collation.set(&my_charset_numeric, DERIVATION_NUMERIC); fix_char_length(DBL_DIG + 8); @@ -5684,6 +5685,8 @@ bool Item_func_get_user_var::fix_length_and_dec() collation.set(m_var_entry->charset(), DERIVATION_IMPLICIT); max_length= MAX_BLOB_WIDTH - 1; set_handler(&type_handler_long_blob); + if (m_var_entry->type_handler()->field_type() == MYSQL_TYPE_GEOMETRY) + set_handler(m_var_entry->type_handler()); break; case DECIMAL_RESULT: collation.set(&my_charset_numeric, DERIVATION_NUMERIC); @@ -5756,7 +5759,7 @@ bool Item_user_var_as_out_param::fix_fields(THD *thd, Item **ref) DBUG_ASSERT(thd->lex->exchange); if (!(entry= get_variable(&thd->user_vars, &org_name, 1))) return TRUE; - entry->type= STRING_RESULT; + entry->set_handler(&type_handler_long_blob); /* Let us set the same collation which is used for loading of fields in LOAD DATA INFILE. @@ -5772,15 +5775,14 @@ bool Item_user_var_as_out_param::fix_fields(THD *thd, Item **ref) void Item_user_var_as_out_param::set_null_value(CHARSET_INFO* cs) { - ::update_hash(entry, TRUE, 0, 0, STRING_RESULT, cs, 0 /* unsigned_arg */); + ::update_hash(entry, TRUE, 0, 0, &type_handler_long_blob, cs); } void Item_user_var_as_out_param::set_value(const char *str, uint length, CHARSET_INFO* cs) { - ::update_hash(entry, FALSE, (void*)str, length, STRING_RESULT, cs, - 0 /* unsigned_arg */); + ::update_hash(entry, FALSE, (void*)str, length, &type_handler_long_blob, cs); } diff --git a/sql/item_func.h b/sql/item_func.h index 1b8aefe005c84..b205d457f1d9a 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -3066,8 +3066,8 @@ class Item_func_set_user_var :public Item_func_user_var String *str_result(String *str); my_decimal *val_decimal_result(my_decimal *); bool is_null_result(); - bool update_hash(void *ptr, size_t length, enum Item_result type, - CHARSET_INFO *cs, bool unsigned_arg); + bool update_hash(void *ptr, size_t length, const Type_handler *th, + CHARSET_INFO *cs); bool send(Protocol *protocol, st_value *buffer); void make_send_field(THD *thd, Send_field *tmp_field); bool check(bool use_result_field); @@ -3825,7 +3825,6 @@ double my_double_round(double value, longlong dec, bool dec_unsigned, extern bool volatile mqh_used; bool update_hash(user_var_entry *entry, bool set_null, void *ptr, size_t length, - Item_result type, CHARSET_INFO *cs, - bool unsigned_arg); + const Type_handler *th, CHARSET_INFO *cs); #endif /* ITEM_FUNC_INCLUDED */ diff --git a/sql/log.cc b/sql/log.cc index 3d3ac9b2b3541..75199000b85e1 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -6794,18 +6794,12 @@ bool MYSQL_BIN_LOG::write(Log_event *event_info, my_bool *with_annotate) BINLOG_USER_VAR_EVENT *user_var_event; get_dynamic(&thd->user_var_events,(uchar*) &user_var_event, i); - /* setting flags for user var log event */ - uchar flags= User_var_log_event::UNDEF_F; - if (user_var_event->unsigned_flag) - flags|= User_var_log_event::UNSIGNED_F; - User_var_log_event e(thd, user_var_event->user_var_event->name.str, user_var_event->user_var_event->name.length, user_var_event->value, user_var_event->length, - user_var_event->type, - user_var_event->charset_number, - flags, + user_var_event->th->user_var_log_event_data_type( + user_var_event->charset_number), using_trans, direct); if (write_event(&e, cache_data, file)) diff --git a/sql/log_event.cc b/sql/log_event.cc index d367575f9e87b..1a3161d49a41e 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -2880,6 +2880,41 @@ XA_prepare_log_event(const char* buf, User_var_log_event methods **************************************************************************/ +bool Log_event_data_type::unpack_optional_attributes(const char *pos, + const char *end) + +{ + for ( ; pos < end; ) + { + switch (*pos) { + case CHUNK_SIGNED: + m_is_unsigned= false; + pos++; + continue; + case CHUNK_UNSIGNED: + m_is_unsigned= true; + pos++; + continue; + case CHUNK_DATA_TYPE_NAME: + { + pos++; + if (pos >= end) + return true; + uint length= (uchar) *pos++; + if (pos + length > end) + return true; + m_data_type_name= {pos, length}; + pos+= length; + continue; + } + default: + break; // Unknown chunk + } + } + return false; +} + + User_var_log_event:: User_var_log_event(const char* buf, uint event_len, const Format_description_log_event* description_event) @@ -2917,11 +2952,8 @@ User_var_log_event(const char* buf, uint event_len, buf+= UV_NAME_LEN_SIZE + name_len; is_null= (bool) *buf; - flags= User_var_log_event::UNDEF_F; // defaults to UNDEF_F if (is_null) { - type= STRING_RESULT; - charset_number= my_charset_bin.number; val_len= 0; val= 0; } @@ -2936,8 +2968,8 @@ User_var_log_event(const char* buf, uint event_len, goto err; } - type= (Item_result) buf[UV_VAL_IS_NULL]; - charset_number= uint4korr(buf + UV_VAL_IS_NULL + UV_VAL_TYPE_SIZE); + m_type= (Item_result) buf[UV_VAL_IS_NULL]; + m_charset_number= uint4korr(buf + UV_VAL_IS_NULL + UV_VAL_TYPE_SIZE); val_len= uint4korr(buf + UV_VAL_IS_NULL + UV_VAL_TYPE_SIZE + UV_CHARSET_NUMBER_SIZE); @@ -2950,20 +2982,14 @@ User_var_log_event(const char* buf, uint event_len, the flags value. Old events will not have this extra byte, thence, - we keep the flags set to UNDEF_F. + we keep m_is_unsigned==false. */ - size_t bytes_read= (val + val_len) - buf_start; - if (bytes_read > event_len) + const char *pos= val + val_len; + if (pos > buf_end || unpack_optional_attributes(pos, buf_end)) { error= true; goto err; } - if ((data_written - bytes_read) > 0) - { - flags= (uint) *(buf + UV_VAL_IS_NULL + UV_VAL_TYPE_SIZE + - UV_CHARSET_NUMBER_SIZE + UV_VAL_LEN_SIZE + - val_len); - } } err: diff --git a/sql/log_event.h b/sql/log_event.h index 28ac4cc78df33..afe4862f152f5 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -57,6 +57,8 @@ #include "rpl_gtid.h" +#include "log_event_data_type.h" + /* Forward declarations */ #ifndef MYSQL_CLIENT class String; @@ -3269,32 +3271,27 @@ class XA_prepare_log_event: public Xid_apply_log_event @section User_var_log_event_binary_format Binary Format */ -class User_var_log_event: public Log_event + +class User_var_log_event: public Log_event, public Log_event_data_type { public: - enum { - UNDEF_F= 0, - UNSIGNED_F= 1 - }; const char *name; size_t name_len; const char *val; size_t val_len; - Item_result type; - uint charset_number; bool is_null; - uchar flags; #ifdef MYSQL_SERVER bool deferred; query_id_t query_id; User_var_log_event(THD* thd_arg, const char *name_arg, size_t name_len_arg, - const char *val_arg, size_t val_len_arg, Item_result type_arg, - uint charset_number_arg, uchar flags_arg, + const char *val_arg, size_t val_len_arg, + const Log_event_data_type &data_type, bool using_trans, bool direct) :Log_event(thd_arg, 0, using_trans), + Log_event_data_type(data_type), name(name_arg), name_len(name_len_arg), val(val_arg), - val_len(val_len_arg), type(type_arg), charset_number(charset_number_arg), - flags(flags_arg), deferred(false) + val_len(val_len_arg), + deferred(false) { is_null= !val; if (direct) diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index c1c5ddf377c68..c20b41e963220 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -2435,7 +2435,7 @@ bool User_var_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) } else { - switch (type) { + switch (m_type) { case REAL_RESULT: double real_val; char real_buf[FMT_G_BUFSIZE(14)]; @@ -2447,8 +2447,7 @@ bool User_var_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) break; case INT_RESULT: char int_buf[22]; - longlong10_to_str(uint8korr(val), int_buf, - ((flags & User_var_log_event::UNSIGNED_F) ? 10 : -10)); + longlong10_to_str(uint8korr(val), int_buf, is_unsigned() ? 10 : -10); if (my_b_printf(&cache, ":=%s%s\n", int_buf, print_event_info->delimiter)) goto err; @@ -2503,7 +2502,7 @@ bool User_var_log_event::print(FILE* file, PRINT_EVENT_INFO* print_event_info) people want to mysqlbinlog|mysql into another server not supporting the character set. But there's not much to do about this and it's unlikely. */ - if (!(cs= get_charset(charset_number, MYF(0)))) + if (!(cs= get_charset(m_charset_number, MYF(0)))) { /* Generate an unusable command (=> syntax error) is probably the best thing we can do here. diff --git a/sql/log_event_data_type.h b/sql/log_event_data_type.h new file mode 100644 index 0000000000000..e3b2039a63c5d --- /dev/null +++ b/sql/log_event_data_type.h @@ -0,0 +1,74 @@ +/* Copyright (c) 2024, MariaDB Corporation. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */ + +#ifndef LOG_EVENT_DATA_TYPE_H +#define LOG_EVENT_DATA_TYPE_H + +class Log_event_data_type +{ +public: + + enum { + CHUNK_SIGNED= 0, + CHUNK_UNSIGNED= 1, + CHUNK_DATA_TYPE_NAME= 2 + }; + +protected: + LEX_CSTRING m_data_type_name; + Item_result m_type; + uint m_charset_number; + bool m_is_unsigned; + +public: + + Log_event_data_type() + :m_data_type_name({NULL,0}), + m_type(STRING_RESULT), + m_charset_number(my_charset_bin.number), + m_is_unsigned(false) + { } + + Log_event_data_type(const LEX_CSTRING &data_type_name_arg, + Item_result type_arg, + uint charset_number_arg, + bool is_unsigned_arg) + :m_data_type_name(data_type_name_arg), + m_type(type_arg), + m_charset_number(charset_number_arg), + m_is_unsigned(is_unsigned_arg) + { } + + const LEX_CSTRING & data_type_name() const + { + return m_data_type_name; + } + Item_result type() const + { + return m_type; + } + uint charset_number() const + { + return m_charset_number; + } + bool is_unsigned() const + { + return m_is_unsigned; + } + + bool unpack_optional_attributes(const char *str, const char *end); +}; + +#endif // LOG_EVENT_DATA_TYPE_H diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 13a981d438117..2a22021cc9590 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -4170,11 +4170,16 @@ bool XA_prepare_log_event::write() #if defined(HAVE_REPLICATION) static bool user_var_append_name_part(THD *thd, String *buf, - const char *name, size_t name_len) + const char *name, size_t name_len, + const LEX_CSTRING &data_type_name) { return buf->append("@") || append_identifier(thd, buf, name, name_len) || - buf->append("="); + buf->append("=") || + (data_type_name.length && + (buf->append("/*") || + buf->append(data_type_name.str, data_type_name.length) || + buf->append("*/"))); } void User_var_log_event::pack_info(Protocol* protocol) @@ -4184,14 +4189,15 @@ void User_var_log_event::pack_info(Protocol* protocol) char buf_mem[FN_REFLEN+7]; String buf(buf_mem, sizeof(buf_mem), system_charset_info); buf.length(0); - if (user_var_append_name_part(protocol->thd, &buf, name, name_len) || + if (user_var_append_name_part(protocol->thd, &buf, name, name_len, + m_data_type_name) || buf.append("NULL")) return; protocol->store(buf.ptr(), buf.length(), &my_charset_bin); } else { - switch (type) { + switch (m_type) { case REAL_RESULT: { double real_val; @@ -4200,7 +4206,8 @@ void User_var_log_event::pack_info(Protocol* protocol) String buf(buf_mem, sizeof(buf_mem), system_charset_info); float8get(real_val, val); buf.length(0); - if (user_var_append_name_part(protocol->thd, &buf, name, name_len) || + if (user_var_append_name_part(protocol->thd, &buf, name, name_len, + m_data_type_name) || buf.append(buf2, my_gcvt(real_val, MY_GCVT_ARG_DOUBLE, MY_GCVT_MAX_FIELD_WIDTH, buf2, NULL))) return; @@ -4213,10 +4220,11 @@ void User_var_log_event::pack_info(Protocol* protocol) char buf_mem[FN_REFLEN + 22]; String buf(buf_mem, sizeof(buf_mem), system_charset_info); buf.length(0); - if (user_var_append_name_part(protocol->thd, &buf, name, name_len) || + if (user_var_append_name_part(protocol->thd, &buf, name, name_len, + m_data_type_name) || buf.append(buf2, longlong10_to_str(uint8korr(val), buf2, - ((flags & User_var_log_event::UNSIGNED_F) ? 10 : -10))-buf2)) + (is_unsigned() ? 10 : -10))-buf2)) return; protocol->store(buf.ptr(), buf.length(), &my_charset_bin); break; @@ -4229,7 +4237,8 @@ void User_var_log_event::pack_info(Protocol* protocol) String str(buf2, sizeof(buf2), &my_charset_bin); buf.length(0); my_decimal((const uchar *) (val + 2), val[0], val[1]).to_string(&str); - if (user_var_append_name_part(protocol->thd, &buf, name, name_len) || + if (user_var_append_name_part(protocol->thd, &buf, name, name_len, + m_data_type_name) || buf.append(buf2)) return; protocol->store(buf.ptr(), buf.length(), &my_charset_bin); @@ -4242,7 +4251,7 @@ void User_var_log_event::pack_info(Protocol* protocol) String buf(buf_mem, sizeof(buf_mem), system_charset_info); CHARSET_INFO *cs; buf.length(0); - if (!(cs= get_charset(charset_number, MYF(0)))) + if (!(cs= get_charset(m_charset_number, MYF(0)))) { if (buf.append("???")) return; @@ -4251,7 +4260,8 @@ void User_var_log_event::pack_info(Protocol* protocol) { size_t old_len; char *beg, *end; - if (user_var_append_name_part(protocol->thd, &buf, name, name_len) || + if (user_var_append_name_part(protocol->thd, &buf, name, name_len, + m_data_type_name) || buf.append("_") || buf.append(cs->csname) || buf.append(" ")) @@ -4299,10 +4309,10 @@ bool User_var_log_event::write() } else { - buf1[1]= type; - int4store(buf1 + 2, charset_number); + buf1[1]= m_type; + int4store(buf1 + 2, m_charset_number); - switch (type) { + switch (m_type) { case REAL_RESULT: float8store(buf2, *(double*) val); break; @@ -4332,15 +4342,28 @@ bool User_var_log_event::write() buf1_length= 10; } + uchar data_type_name_chunk_signature= (uchar) CHUNK_DATA_TYPE_NAME; + uint data_type_name_chunk_signature_length= m_data_type_name.length ? 1 : 0; + uchar data_type_name_length_length= m_data_type_name.length ? 1 : 0; + /* Length of the whole event */ - event_length= sizeof(buf)+ name_len + buf1_length + val_len + unsigned_len; + event_length= sizeof(buf)+ name_len + buf1_length + val_len + unsigned_len + + data_type_name_chunk_signature_length + + data_type_name_length_length + + (uint) m_data_type_name.length; + uchar unsig= m_is_unsigned ? CHUNK_UNSIGNED : CHUNK_SIGNED; + uchar data_type_name_length= (uchar) m_data_type_name.length; return write_header(event_length) || write_data(buf, sizeof(buf)) || write_data(name, name_len) || write_data(buf1, buf1_length) || write_data(pos, val_len) || - write_data(&flags, unsigned_len) || + write_data(&unsig, unsigned_len) || + write_data(&data_type_name_chunk_signature, + data_type_name_chunk_signature_length) || + write_data(&data_type_name_length, data_type_name_length_length) || + write_data(m_data_type_name.str, (uint) m_data_type_name.length) || write_footer(); } @@ -4364,7 +4387,7 @@ int User_var_log_event::do_apply_event(rpl_group_info *rgi) current_thd->query_id= query_id; /* recreating original time context */ } - if (!(charset= get_charset(charset_number, MYF(MY_WME)))) + if (!(charset= get_charset(m_charset_number, MYF(MY_WME)))) { rgi->rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, ER_THD(thd, ER_SLAVE_FATAL_ERROR), @@ -4383,7 +4406,7 @@ int User_var_log_event::do_apply_event(rpl_group_info *rgi) } else { - switch (type) { + switch (m_type) { case REAL_RESULT: if (val_len != 8) { @@ -4447,13 +4470,10 @@ int User_var_log_event::do_apply_event(rpl_group_info *rgi) if (e->fix_fields(thd, 0)) DBUG_RETURN(1); - /* - A variable can just be considered as a table with - a single record and with a single column. Thus, like - a column value, it could always have IMPLICIT derivation. - */ - e->update_hash((void*) val, val_len, type, charset, - (flags & User_var_log_event::UNSIGNED_F)); + const Type_handler *th= Type_handler::handler_by_log_event_data_type(thd, + *this); + e->update_hash((void*) val, val_len, th, charset); + if (!is_deferred()) free_root(thd->mem_root, 0); else diff --git a/sql/sql_binlog.cc b/sql/sql_binlog.cc index 4dd5f16f351d7..05ae01b542faa 100644 --- a/sql/sql_binlog.cc +++ b/sql/sql_binlog.cc @@ -134,7 +134,7 @@ int binlog_defragment(THD *thd) entry[k]= (user_var_entry*) my_hash_search(&thd->user_vars, (uchar*) name[k].str, name[k].length); - if (!entry[k] || entry[k]->type != STRING_RESULT) + if (!entry[k] || entry[k]->type_handler()->result_type() != STRING_RESULT) { my_error(ER_WRONG_TYPE_FOR_VAR, MYF(0), name[k].str); return -1; @@ -159,7 +159,8 @@ int binlog_defragment(THD *thd) gathered_length += entry[k]->length; } for (uint k=0; k < 2; k++) - update_hash(entry[k], true, NULL, 0, STRING_RESULT, &my_charset_bin, 0); + update_hash(entry[k], true, NULL, 0, + &type_handler_long_blob, &my_charset_bin); DBUG_ASSERT(gathered_length == thd->lex->comment.length); diff --git a/sql/sql_class.h b/sql/sql_class.h index 06abab99be733..a8ef864821028 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -275,9 +275,8 @@ typedef struct st_user_var_events user_var_entry *user_var_event; char *value; size_t length; - Item_result type; + const Type_handler *th; uint charset_number; - bool unsigned_flag; } BINLOG_USER_VAR_EVENT; /* @@ -6808,7 +6807,7 @@ class Qualified_column_ident: public Table_ident // this is needed for user_vars hash -class user_var_entry +class user_var_entry: public Type_handler_hybrid_field_type { CHARSET_INFO *m_charset; public: @@ -6817,8 +6816,6 @@ class user_var_entry char *value; size_t length; query_id_t update_query_id, used_query_id; - Item_result type; - bool unsigned_flag; double val_real(bool *null_value); longlong val_int(bool *null_value) const; diff --git a/sql/sql_repl.cc b/sql/sql_repl.cc index 1a83b96b61bfe..270f051b9dd2a 100644 --- a/sql/sql_repl.cc +++ b/sql/sql_repl.cc @@ -510,7 +510,7 @@ static enum enum_binlog_checksum_alg get_binlog_checksum_value_at_connect(THD * } else { - DBUG_ASSERT(entry->type == STRING_RESULT); + DBUG_ASSERT(entry->type_handler()->result_type() == STRING_RESULT); String str; uint dummy_errors; str.copy(entry->value, entry->length, &my_charset_bin, &my_charset_bin, diff --git a/sql/sql_type.cc b/sql/sql_type.cc index 13e61a40be2dd..77931b6cf16fb 100644 --- a/sql/sql_type.cc +++ b/sql/sql_type.cc @@ -2248,6 +2248,34 @@ Type_handler::get_handler_by_real_type(enum_field_types type) } +const Type_handler * +Type_handler::handler_by_log_event_data_type(THD *thd, + const Log_event_data_type &type) +{ + if (type.data_type_name().length) + { + const Type_handler *th= handler_by_name(thd, type.data_type_name()); + if (th) + return th; + } + switch (type.type()) { + case STRING_RESULT: + case ROW_RESULT: + case TIME_RESULT: + break; + case REAL_RESULT: + return &type_handler_double; + case INT_RESULT: + if (type.is_unsigned()) + return &type_handler_ulonglong; + return &type_handler_slonglong; + case DECIMAL_RESULT: + return &type_handler_newdecimal; + } + return &type_handler_long_blob; +} + + /** Create a DOUBLE field by default. */ diff --git a/sql/sql_type.h b/sql/sql_type.h index edce7f45abe1b..c8c92e5625370 100644 --- a/sql/sql_type.h +++ b/sql/sql_type.h @@ -30,6 +30,8 @@ #include "sql_type_string.h" #include "sql_type_real.h" #include "compat56.h" +#include "log_event_data_type.h" + C_MODE_START #include C_MODE_END @@ -3652,6 +3654,9 @@ class Type_handler static const Type_handler *handler_by_name(THD *thd, const LEX_CSTRING &name); static const Type_handler *handler_by_name_or_error(THD *thd, const LEX_CSTRING &name); + static const Type_handler *handler_by_log_event_data_type( + THD *thd, + const Log_event_data_type &type); static const Type_handler *odbc_literal_type_handler(const LEX_CSTRING *str); static const Type_handler *blob_type_handler(uint max_octet_length); static const Type_handler *string_type_handler(uint max_octet_length); @@ -3936,6 +3941,12 @@ class Type_handler { return false; } + + virtual Log_event_data_type user_var_log_event_data_type(uint charset_nr) const + { + return Log_event_data_type({NULL,0}/*data type name*/, result_type(), + charset_nr, is_unsigned()); + } virtual uint Column_definition_gis_options_image(uchar *buff, const Column_definition &def) const diff --git a/sql/sql_type_geom.h b/sql/sql_type_geom.h index ff00beea598ff..858c2da231846 100644 --- a/sql/sql_type_geom.h +++ b/sql/sql_type_geom.h @@ -81,6 +81,13 @@ class Type_handler_geometry: public Type_handler_string_result Field *make_conversion_table_field(MEM_ROOT *root, TABLE *table, uint metadata, const Field *target) const override; + Log_event_data_type user_var_log_event_data_type(uint charset_nr) + const override + { + return Log_event_data_type(name().lex_cstring(), result_type(), + charset_nr, false/*unsigned*/); + } + uint Column_definition_gis_options_image(uchar *buff, const Column_definition &def) const override; diff --git a/storage/spider/spd_conn.cc b/storage/spider/spd_conn.cc index 9897e495c8377..befd9bfd2db36 100644 --- a/storage/spider/spd_conn.cc +++ b/storage/spider/spd_conn.cc @@ -1886,7 +1886,7 @@ int spider_conn_queue_loop_check( loop_check_buf[lex_str.length] = '\0'; DBUG_PRINT("info", ("spider param name=%s", lex_str.str)); loop_check = get_variable(&thd->user_vars, &lex_str, FALSE); - if (!loop_check || loop_check->type != STRING_RESULT) + if (!loop_check || loop_check->type_handler()->result_type() != STRING_RESULT) { DBUG_PRINT("info", ("spider client is not Spider")); lex_str.str = ""; diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc index 4ff54b591ba57..717313df2af95 100644 --- a/storage/spider/spd_table.cc +++ b/storage/spider/spd_table.cc @@ -4742,7 +4742,7 @@ SPIDER_SHARE *spider_get_share( ((char *) lex_str.str)[lex_str.length] = '\0'; DBUG_PRINT("info",("spider loop check param name=%s", lex_str.str)); loop_check = get_variable(&thd->user_vars, &lex_str, FALSE); - if (loop_check && loop_check->type == STRING_RESULT) + if (loop_check && loop_check->type_handler()->result_type() == STRING_RESULT) { lex_str.length = top_share->path.length + spider_unique_id.length + 1; lex_str.str = loop_check_buf + buf_sz - top_share->path.length - From f8c88d905b44bffe161158309e9acc25ad3691aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 17 Jan 2024 11:14:24 +0200 Subject: [PATCH 30/33] MDEV-33213 History list is not shrunk unless there is a pause in the workload The parameter innodb_undo_log_truncate=ON enables a multi-phased logic: 1. Any "producers" (new starting transactions) are prohibited from using the rollback segments that reside in the undo tablespace. 2. Any transactions that use any of the rollback segments must be committed or aborted. 3. The purge of committed transaction history must process all the rollback segments. 4. The undo tablespace is truncated and rebuilt. 5. The rollback segments are re-enabled for new transactions. There was one flaw in this logic: The first step was not being invoked as often as it could be, and therefore innodb_undo_log_truncate=ON would have no chance to work during a heavy write workload. Independent of innodb_undo_log_truncate, even after commit 86767bcc0f121db3ad83a74647a642754a0ce57f we are missing some chances to free processed undo log pages. If we prohibited the creation of new transactions in one busy rollback segment at a time, we would be eventually guaranteed to be able to free such pages. purge_sys_t::skipped_rseg: The current candidate rollback segment for shrinking the history independent of innodb_undo_log_truncate. purge_sys_t::iterator::free_history_rseg(): Renamed from trx_purge_truncate_rseg_history(). Implement the logic around purge_sys.m_skipped_rseg. purge_sys_t::truncate_undo_space: Renamed from truncate. purge_sys.truncate_undo_space.last: Changed the type to integer to get rid of some pointer dereferencing and conditional branches. purge_sys_t::truncating_tablespace(), purge_sys_t::undo_truncate_try(): Refactored from trx_purge_truncate_history(). Set purge_sys.truncate_undo_space.current if applicable, or return an already set purge_sys.truncate_undo_space.current. purge_coordinator_state::do_purge(): Invoke purge_sys_t::truncating_tablespace() as part of the normal work loop, to implement innodb_undo_log_truncate=ON as often as possible. trx_purge_truncate_rseg_history(): Remove a redundant parameter. trx_undo_truncate_start(): Replace dead code with a debug assertion. Correctness tested by: Matthias Leich Performance tested by: Axel Schwenke Reviewed by: Debarun Banerjee --- storage/innobase/include/trx0purge.h | 51 +++++- storage/innobase/include/trx0rseg.h | 5 +- storage/innobase/include/trx0sys.h | 5 + storage/innobase/srv/srv0srv.cc | 3 +- storage/innobase/trx/trx0purge.cc | 235 ++++++++++++++++++--------- storage/innobase/trx/trx0undo.cc | 8 +- 6 files changed, 213 insertions(+), 94 deletions(-) diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index 3ddd2e98fee8b..a836c134c0287 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -140,6 +140,15 @@ class purge_sys_t bool m_initialized{false}; /** whether purge is enabled; protected by latch and std::atomic */ std::atomic m_enabled{false}; + /** The primary candidate for iterator::free_history() is + rseg=trx_sys.rseg_array[skipped_rseg]. This field may be changed + after invoking rseg.set_skip_allocation() and rseg.clear_skip_allocation() + and while holding the exclusive rseg.latch. + + This may only be 0 if innodb_undo_tablespaces=0, because rollback segment + 0 always resides in the system tablespace and would never be used when + dedicated undo tablespaces are in use. */ + Atomic_relaxed skipped_rseg; public: /** whether purge is active (may hold table handles) */ std::atomic m_active{false}; @@ -197,6 +206,11 @@ class purge_sys_t return undo_no <= other.undo_no; } + /** Remove unnecessary history data from a rollback segment. + @param rseg rollback segment + @return error code */ + inline dberr_t free_history_rseg(trx_rseg_t &rseg) const; + /** Free the undo pages up to this. */ dberr_t free_history() const; @@ -240,14 +254,15 @@ class purge_sys_t by the pq_mutex */ mysql_mutex_t pq_mutex; /*!< Mutex protecting purge_queue */ - /** Undo tablespace file truncation (only accessed by the - srv_purge_coordinator_thread) */ - struct { - /** The undo tablespace that is currently being truncated */ - fil_space_t* current; - /** The undo tablespace that was last truncated */ - fil_space_t* last; - } truncate; + /** innodb_undo_log_truncate=ON state; + only modified by purge_coordinator_callback() */ + struct { + /** The undo tablespace that is currently being truncated */ + Atomic_relaxed current; + /** The number of the undo tablespace that was last truncated, + relative from srv_undo_space_id_start */ + ulint last; + } truncate_undo_space; /** Create the instance */ void create(); @@ -357,6 +372,26 @@ class purge_sys_t typically via purge_sys_t::view_guard. */ return view.sees(id); } + +private: + /** Enable the use of a rollback segment and advance skipped_rseg, + after iterator::free_history_rseg() had invoked + rseg.set_skip_allocation(). */ + inline void rseg_enable(trx_rseg_t &rseg); + + /** Try to start truncating a tablespace. + @param id undo tablespace identifier + @param size the maximum desired undo tablespace size, in pages + @return undo tablespace whose truncation was started + @retval nullptr if truncation is not currently possible */ + inline fil_space_t *undo_truncate_try(ulint id, ulint size); +public: + /** Check if innodb_undo_log_truncate=ON needs to be handled. + This is only to be called by purge_coordinator_callback(). + @return undo tablespace chosen by innodb_undo_log_truncate=ON + @retval nullptr if truncation is not currently possible */ + fil_space_t *truncating_tablespace(); + /** A wrapper around trx_sys_t::clone_oldest_view(). */ template void clone_oldest_view() diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 1d95b7d2e7a73..f5d9a2199a153 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -73,14 +73,15 @@ struct alignas(CPU_LEVEL1_DCACHE_LINESIZE) trx_rseg_t /** Reference counter to track is_persistent() transactions, with SKIP flag. */ std::atomic ref; - +public: /** Whether undo tablespace truncation is pending */ static constexpr uint32_t SKIP= 1; /** Transaction reference count multiplier */ static constexpr uint32_t REF= 2; + /** @return the reference count and flags */ uint32_t ref_load() const { return ref.load(std::memory_order_relaxed); } - +private: /** Set the SKIP bit */ void ref_set_skip() { diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index c5f8ab012c455..cf1579a3e3098 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -1189,6 +1189,11 @@ class trx_sys_t return count; } + /** Disable further allocation of transactions in a rollback segment + that are subject to innodb_undo_log_truncate=ON + @param space undo tablespace that will be truncated */ + inline void undo_truncate_start(fil_space_t &space); + private: static my_bool find_same_or_older_callback(rw_trx_hash_element_t *element, trx_id_t *id) diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 7923b14d69d55..b29caf255fc69 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1638,7 +1638,8 @@ inline void purge_coordinator_state::do_purge() ulint n_pages_handled= trx_purge(n_threads, history_size); if (!trx_sys.history_exists()) goto no_history; - if (purge_sys.truncate.current || srv_shutdown_state != SRV_SHUTDOWN_NONE) + if (purge_sys.truncating_tablespace() || + srv_shutdown_state != SRV_SHUTDOWN_NONE) { purge_truncation_task.wait(); trx_purge_truncate_history(); diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 230036c849959..30cce66697f01 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -169,10 +169,15 @@ void purge_sys_t::create() ut_ad(this == &purge_sys); ut_ad(!m_initialized); ut_ad(!enabled()); + ut_ad(!m_active); + /* If innodb_undo_tablespaces>0, the rollback segment 0 + (which always resides in the system tablespace) will + never be used; @see trx_assign_rseg_low() */ + skipped_rseg= srv_undo_tablespaces > 0; m_paused= 0; query= purge_graph_build(); next_stored= false; - rseg= NULL; + rseg= nullptr; page_no= 0; offset= 0; hdr_page_no= 0; @@ -180,8 +185,8 @@ void purge_sys_t::create() latch.SRW_LOCK_INIT(trx_purge_latch_key); end_latch.init(); mysql_mutex_init(purge_sys_pq_mutex_key, &pq_mutex, nullptr); - truncate.current= NULL; - truncate.last= NULL; + truncate_undo_space.current= nullptr; + truncate_undo_space.last= 0; m_initialized= true; } @@ -387,17 +392,50 @@ static void trx_purge_free_segment(buf_block_t *rseg_hdr, buf_block_t *block, block->page.frame, &mtr)); } +void purge_sys_t::rseg_enable(trx_rseg_t &rseg) +{ + ut_ad(this == &purge_sys); +#ifndef SUX_LOCK_GENERIC + ut_ad(rseg.latch.is_write_locked()); +#endif + uint8_t skipped= skipped_rseg; + ut_ad(skipped < TRX_SYS_N_RSEGS); + if (&rseg == &trx_sys.rseg_array[skipped]) + { + /* If this rollback segment is subject to innodb_undo_log_truncate=ON, + we must not clear the flag. But we will advance purge_sys.skipped_rseg + to be able to choose another candidate for this soft truncation, and + to prevent the following scenario: + + (1) purge_sys_t::iterator::free_history_rseg() had invoked + rseg.set_skip_allocation() + (2) undo log truncation had completed on this rollback segment + (3) SET GLOBAL innodb_undo_log_truncate=OFF + (4) purge_sys_t::iterator::free_history_rseg() would not be able to + invoke rseg.set_skip_allocation() on any other rollback segment + before this rseg has grown enough */ + if (truncate_undo_space.current != rseg.space) + rseg.clear_skip_allocation(); + skipped++; + /* If innodb_undo_tablespaces>0, the rollback segment 0 + (which always resides in the system tablespace) will + never be used; @see trx_assign_rseg_low() */ + if (!(skipped%= TRX_SYS_N_RSEGS) && srv_undo_tablespaces) + skipped++; + skipped_rseg= skipped; + } +} + /** Remove unnecessary history data from a rollback segment. @param rseg rollback segment @param limit truncate anything before this -@param all whether everything can be truncated @return error code */ -static dberr_t -trx_purge_truncate_rseg_history(trx_rseg_t &rseg, - const purge_sys_t::iterator &limit, bool all) +inline dberr_t purge_sys_t::iterator::free_history_rseg(trx_rseg_t &rseg) const { fil_addr_t hdr_addr; mtr_t mtr; + bool freed= false; + uint32_t rseg_ref= 0; mtr.start(); @@ -407,6 +445,8 @@ trx_purge_truncate_rseg_history(trx_rseg_t &rseg, { func_exit: mtr.commit(); + if (freed && (rseg.SKIP & rseg_ref)) + purge_sys.rseg_enable(rseg); return err; } @@ -428,16 +468,40 @@ trx_purge_truncate_rseg_history(trx_rseg_t &rseg, const trx_id_t undo_trx_no= mach_read_from_8(b->page.frame + hdr_addr.boffset + TRX_UNDO_TRX_NO); - if (undo_trx_no >= limit.trx_no) + if (undo_trx_no >= trx_no) { - if (undo_trx_no == limit.trx_no) - err = trx_undo_truncate_start(&rseg, hdr_addr.page, - hdr_addr.boffset, limit.undo_no); + if (undo_trx_no == trx_no) + err= trx_undo_truncate_start(&rseg, hdr_addr.page, + hdr_addr.boffset, undo_no); goto func_exit; } - - if (!all) - goto func_exit; + else + { + rseg_ref= rseg.ref_load(); + if (rseg_ref >= rseg.REF || !purge_sys.sees(rseg.needs_purge)) + { + /* We cannot clear this entire rseg because trx_assign_rseg_low() + has already chosen it for a future trx_undo_assign(), or + because some recently started transaction needs purging. + + If this invocation could not reduce rseg.history_size at all + (!freed), we will try to ensure progress and prevent our + starvation by disabling one rollback segment for future + trx_assign_rseg_low() invocations until a future invocation has + made progress and invoked purge_sys_t::rseg_enable(rseg) on that + rollback segment. */ + + if (!(rseg.SKIP & rseg_ref) && !freed && + ut_d(!trx_rseg_n_slots_debug &&) + &rseg == &trx_sys.rseg_array[purge_sys.skipped_rseg]) + /* If rseg.space == purge_sys.truncate_undo_space.current + the following will be a no-op. A possible conflict + with innodb_undo_log_truncate=ON will be handled in + purge_sys_t::rseg_enable(). */ + rseg.set_skip_allocation(); + goto func_exit; + } + } fil_addr_t prev_hdr_addr= flst_get_prev_addr(b->page.frame + hdr_addr.boffset + @@ -500,6 +564,7 @@ trx_purge_truncate_rseg_history(trx_rseg_t &rseg, mtr.commit(); ut_ad(rseg.history_size > 0); rseg.history_size--; + freed= true; mtr.start(); rseg_hdr->page.lock.x_lock(); ut_ad(rseg_hdr->page.id() == rseg.page_id()); @@ -554,9 +619,7 @@ dberr_t purge_sys_t::iterator::free_history() const ut_ad(rseg.is_persistent()); log_free_check(); rseg.latch.wr_lock(SRW_LOCK_CALL); - dberr_t err= - trx_purge_truncate_rseg_history(rseg, *this, !rseg.is_referenced() && - purge_sys.sees(rseg.needs_purge)); + dberr_t err= free_history_rseg(rseg); rseg.latch.wr_unlock(); if (err) return err; @@ -564,6 +627,62 @@ dberr_t purge_sys_t::iterator::free_history() const return DB_SUCCESS; } +inline void trx_sys_t::undo_truncate_start(fil_space_t &space) +{ + ut_ad(this == &trx_sys); + /* Undo tablespace always are a single file. */ + ut_a(UT_LIST_GET_LEN(space.chain) == 1); + fil_node_t *file= UT_LIST_GET_FIRST(space.chain); + /* The undo tablespace files are never closed. */ + ut_ad(file->is_open()); + sql_print_information("InnoDB: Starting to truncate %s", file->name); + + for (auto &rseg : rseg_array) + if (rseg.space == &space) + { + /* Prevent a race with purge_sys_t::iterator::free_history_rseg() */ + rseg.latch.rd_lock(SRW_LOCK_CALL); + /* Once set, this rseg will not be allocated to subsequent + transactions, but we will wait for existing active + transactions to finish. */ + rseg.set_skip_allocation(); + rseg.latch.rd_unlock(); + } +} + +inline fil_space_t *purge_sys_t::undo_truncate_try(ulint id, ulint size) +{ + ut_ad(srv_is_undo_tablespace(id)); + fil_space_t *space= fil_space_get(id); + if (space && space->get_size() > size) + { + truncate_undo_space.current= space; + trx_sys.undo_truncate_start(*space); + return space; + } + return nullptr; +} + +fil_space_t *purge_sys_t::truncating_tablespace() +{ + ut_ad(this == &purge_sys); + + fil_space_t *space= truncate_undo_space.current; + if (space || srv_undo_tablespaces_active < 2 || !srv_undo_log_truncate) + return space; + + const ulint size= ulint(srv_max_undo_log_size >> srv_page_size_shift); + for (ulint i= truncate_undo_space.last, j= i;; ) + { + if (fil_space_t *s= undo_truncate_try(srv_undo_space_id_start + i, size)) + return s; + ++i; + i%= srv_undo_tablespaces_active; + if (i == j) + return nullptr; + } +} + #if defined __GNUC__ && __GNUC__ == 4 && !defined __clang__ # if defined __arm__ || defined __aarch64__ /* Work around an internal compiler error in GCC 4.8.5 */ @@ -589,55 +708,14 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() head.undo_no= 0; } - if (head.free_history() != DB_SUCCESS || srv_undo_tablespaces_active < 2) + if (head.free_history() != DB_SUCCESS) return; - while (srv_undo_log_truncate) + while (fil_space_t *space= purge_sys.truncating_tablespace()) { - if (!purge_sys.truncate.current) - { - const ulint threshold= - ulint(srv_max_undo_log_size >> srv_page_size_shift); - for (ulint i= purge_sys.truncate.last - ? purge_sys.truncate.last->id - srv_undo_space_id_start : 0, - j= i;; ) - { - const auto space_id= srv_undo_space_id_start + i; - ut_ad(srv_is_undo_tablespace(space_id)); - fil_space_t *space= fil_space_get(space_id); - ut_a(UT_LIST_GET_LEN(space->chain) == 1); - - if (space && space->get_size() > threshold) - { - purge_sys.truncate.current= space; - break; - } - - ++i; - i %= srv_undo_tablespaces_active; - if (i == j) - return; - } - } - - fil_space_t &space= *purge_sys.truncate.current; - /* Undo tablespace always are a single file. */ - fil_node_t *file= UT_LIST_GET_FIRST(space.chain); - /* The undo tablespace files are never closed. */ - ut_ad(file->is_open()); - - DBUG_LOG("undo", "marking for truncate: " << file->name); - - for (auto &rseg : trx_sys.rseg_array) - if (rseg.space == &space) - /* Once set, this rseg will not be allocated to subsequent - transactions, but we will wait for existing active - transactions to finish. */ - rseg.set_skip_allocation(); - for (auto &rseg : trx_sys.rseg_array) { - if (rseg.space != &space) + if (rseg.space != space) continue; rseg.latch.rd_lock(SRW_LOCK_CALL); @@ -670,8 +748,9 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() rseg.latch.rd_unlock(); } - sql_print_information("InnoDB: Truncating %s", file->name); - trx_purge_cleanse_purge_queue(space); + const char *file_name= UT_LIST_GET_FIRST(space->chain)->name; + sql_print_information("InnoDB: Truncating %s", file_name); + trx_purge_cleanse_purge_queue(*space); /* Lock all modified pages of the tablespace. @@ -688,12 +767,12 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() /* Adjust the tablespace metadata. */ mysql_mutex_lock(&fil_system.mutex); - if (space.crypt_data) + if (space->crypt_data) { - space.reacquire(); + space->reacquire(); mysql_mutex_unlock(&fil_system.mutex); - fil_space_crypt_close_tablespace(&space); - space.release(); + fil_space_crypt_close_tablespace(space); + space->release(); } else mysql_mutex_unlock(&fil_system.mutex); @@ -705,17 +784,17 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() mtr_t mtr; mtr.start(); - mtr.x_lock_space(&space); + mtr.x_lock_space(space); /* Associate the undo tablespace with mtr. During mtr::commit_shrink(), InnoDB can use the undo tablespace object to clear all freed ranges */ - mtr.set_named_space(&space); - mtr.trim_pages(page_id_t(space.id, size)); - ut_a(fsp_header_init(&space, size, &mtr) == DB_SUCCESS); + mtr.set_named_space(space); + mtr.trim_pages(page_id_t(space->id, size)); + ut_a(fsp_header_init(space, size, &mtr) == DB_SUCCESS); for (auto &rseg : trx_sys.rseg_array) { - if (rseg.space != &space) + if (rseg.space != space) continue; ut_ad(!rseg.is_referenced()); @@ -724,7 +803,7 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() possibly before this server had been started up. */ dberr_t err; - buf_block_t *rblock= trx_rseg_header_create(&space, + buf_block_t *rblock= trx_rseg_header_create(space, &rseg - trx_sys.rseg_array, trx_sys.get_max_trx_id(), &mtr, &err); @@ -737,7 +816,7 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() rseg.reinit(rblock->page.id().page_no()); } - mtr.commit_shrink(space, size); + mtr.commit_shrink(*space, size); /* No mutex; this is only updated by the purge coordinator. */ export_vars.innodb_undo_truncations++; @@ -759,10 +838,10 @@ TRANSACTIONAL_TARGET void trx_purge_truncate_history() log_buffer_flush_to_disk(); DBUG_SUICIDE();); - sql_print_information("InnoDB: Truncated %s", file->name); - purge_sys.truncate.last= purge_sys.truncate.current; - ut_ad(&space == purge_sys.truncate.current); - purge_sys.truncate.current= nullptr; + sql_print_information("InnoDB: Truncated %s", file_name); + ut_ad(space == purge_sys.truncate_undo_space.current); + purge_sys.truncate_undo_space.current= nullptr; + purge_sys.truncate_undo_space.last= space->id - srv_undo_space_id_start; } } diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index def47686da243..ccc68dfeef774 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -907,15 +907,13 @@ trx_undo_truncate_start( trx_undo_rec_t* last_rec; mtr_t mtr; + ut_ad(rseg->is_persistent()); + if (!limit) { return DB_SUCCESS; } loop: - mtr_start(&mtr); - - if (!rseg->is_persistent()) { - mtr.set_log_mode(MTR_LOG_NO_REDO); - } + mtr.start(); dberr_t err; const buf_block_t* undo_page; From 6a514ef6727c1c1102f57554cdf28aaa775d210a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 17 Jan 2024 12:50:44 +0200 Subject: [PATCH 31/33] MDEV-30940: Try to fix the test --- mysql-test/suite/innodb/r/lock_move_wait_lock_race.result | 3 ++- mysql-test/suite/innodb/t/lock_move_wait_lock_race.test | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/lock_move_wait_lock_race.result b/mysql-test/suite/innodb/r/lock_move_wait_lock_race.result index 572fbc9b1d191..c78102d9c3b1d 100644 --- a/mysql-test/suite/innodb/r/lock_move_wait_lock_race.result +++ b/mysql-test/suite/innodb/r/lock_move_wait_lock_race.result @@ -1,4 +1,5 @@ -CREATE TABLE t (pk int PRIMARY KEY, c varchar(10)) ENGINE=InnoDB; +CREATE TABLE t (pk int PRIMARY KEY, c varchar(10)) +STATS_PERSISTENT=0 ENGINE=InnoDB; INSERT INTO t VALUES (10, "0123456789"); connection default; BEGIN; diff --git a/mysql-test/suite/innodb/t/lock_move_wait_lock_race.test b/mysql-test/suite/innodb/t/lock_move_wait_lock_race.test index 3a04c7127c821..0f88f8d9d0f09 100644 --- a/mysql-test/suite/innodb/t/lock_move_wait_lock_race.test +++ b/mysql-test/suite/innodb/t/lock_move_wait_lock_race.test @@ -3,7 +3,8 @@ --source include/have_debug.inc --source include/have_debug_sync.inc -CREATE TABLE t (pk int PRIMARY KEY, c varchar(10)) ENGINE=InnoDB; +CREATE TABLE t (pk int PRIMARY KEY, c varchar(10)) +STATS_PERSISTENT=0 ENGINE=InnoDB; INSERT INTO t VALUES (10, "0123456789"); --connection default From 03854a84abf71c734f7a1c49897e3ef010a3fe4e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 17 Jan 2024 22:26:12 +0100 Subject: [PATCH 32/33] MDEV-32374 Improve lsn_lock. Also use futex-like on Windows Upon further benchmarking, it turns out srw_mutex performs overall slightly better with WaitOnAddress than CRITICAL_SECTION. --- storage/innobase/include/log0log.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 58213856c791b..54851ca0a6563 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -180,9 +180,6 @@ struct log_t /* On ARM, we do more spinning */ typedef srw_spin_lock log_rwlock; typedef pthread_mutex_wrapper log_lsn_lock; -#elif defined _WIN32 - typedef srw_lock log_rwlock; - typedef pthread_mutex_wrapper log_lsn_lock; #else typedef srw_lock log_rwlock; typedef srw_mutex log_lsn_lock; From f63045b11950c0659b7e3dcf66996be0422d76dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 18 Jan 2024 10:14:21 +0200 Subject: [PATCH 33/33] MDEV-33213 fixup: GCC 5 -Wconversion --- storage/innobase/trx/trx0purge.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 30cce66697f01..7eaedabdd9039 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -420,7 +420,7 @@ void purge_sys_t::rseg_enable(trx_rseg_t &rseg) /* If innodb_undo_tablespaces>0, the rollback segment 0 (which always resides in the system tablespace) will never be used; @see trx_assign_rseg_low() */ - if (!(skipped%= TRX_SYS_N_RSEGS) && srv_undo_tablespaces) + if (!(skipped&= (TRX_SYS_N_RSEGS - 1)) && srv_undo_tablespaces) skipped++; skipped_rseg= skipped; }