From b015dd3f3a75e50995443dd203d37a927126998b Mon Sep 17 00:00:00 2001 From: Rahul Singhal Date: Fri, 30 Oct 2020 14:34:50 -0700 Subject: [PATCH] Added COM_SEND_REPLICA_STATISTICS & background thread to send lag statistics from secondary to primary and store it in information_schema.replica_statistics Summary: This diff is a port of D21440060 (https://github.com/facebook/mysql-5.6/commit/06e73673b562438e99f83f43a3fba069d5a7e4d8) (https://github.com/facebook/mysql-5.6/commit/06e73673b562438e99f83f43a3fba069d5a7e4d8) from mysql-5.6.35 to mysql-8.0 * I am adding a new RPC `COM_SEND_REPLICA_STATISTICS`. Slaves will use this RPC to send lag statistics to master. This will be done in the next diff. * In addition, I have added the information_schema table named `replica_statistics` to store the slave lag stats. * Added a new background thread that is started when the `mysqld` process starts and continuously publishes lag statistics from slaves to master every `write_stats_frequency` seconds. The default values of `write_stats_frequency` is set to 0, which means do not send lag statistics to master. The unit for this sys_var is seconds. Also added another sys_var `write_stats_count` to control the number of data points to cache in replica_statistics table per secondary **Points to note -** * The background thread re-uses the connection to master to send stats. It does not reconnect every cycle. * If it is not able to connect to the master in one cycle, it retries the connection in successive cycles until it is able to connect. After this point, it reuses the same connection. * In case of topology changes, the thread is able to reconnect to the new master and send stats. Differential Revision: D24659846 --- client/check/mysqlcheck_core.cc | 2 +- client/client_priv.h | 2 +- .../mysqldump_tool_chain_maker_options.cc | 2 +- client/mysqldump.cc | 8 +- include/map_helpers.h | 13 +- include/my_command.h | 3 + include/mysql.h.pp | 3 + include/mysql/plugin_audit.h.pp | 3 + mysql-test/include/check-testcase.test | 1 + mysql-test/r/disabled_replication.result | 2 +- mysql-test/r/mysqld--help-notwin.result | 12 +- mysql-test/r/slave_stats_daemon.result | 197 ++++++++++++ .../suite/perfschema/r/all_tests.result | 1 + .../perfschema/r/dd_version_check.result | 4 +- .../r/ddl_replica_statistics.result | 11 + .../suite/perfschema/r/dml_handler.result | 87 ++--- .../r/dml_replica_statistics.result | 19 ++ .../perfschema/r/dml_setup_instruments.result | 2 +- .../perfschema/r/information_schema.result | 10 + mysql-test/suite/perfschema/r/schema.result | 1 + .../suite/perfschema/r/table_schema.result | 3 + .../suite/perfschema/t/dd_version_check.test | 4 + .../perfschema/t/ddl_replica_statistics.test | 21 ++ .../perfschema/t/dml_replica_statistics.test | 35 ++ .../sys_vars/r/write_stats_count_basic.result | 27 ++ .../r/write_stats_frequency_basic.result | 27 ++ .../sys_vars/t/write_stats_count_basic.test | 44 +++ .../t/write_stats_frequency_basic.test | 44 +++ mysql-test/t/slave_stats_daemon.test | 223 +++++++++++++ sql-common/client.cc | 3 +- sql-common/net_serv.cc | 2 +- sql/CMakeLists.txt | 1 + sql/json_dom.cc | 1 + sql/log.cc | 2 +- sql/mysqld.cc | 24 +- sql/mysqld.h | 8 +- sql/protocol_classic.cc | 2 +- sql/rpl_replica.cc | 299 +++++++++++------- sql/rpl_replica.h | 3 + sql/rpl_source.cc | 69 ++++ sql/rpl_source.h | 19 ++ sql/slave_stats_daemon.cc | 249 +++++++++++++++ sql/slave_stats_daemon.h | 29 ++ sql/sql_audit.cc | 3 +- sql/sql_class.h | 3 +- sql/sql_parse.cc | 234 +++++++++++++- sql/sql_parse.h | 4 +- sql/sys_vars.cc | 32 +- sql/table.cc | 9 + sql/table.h | 11 +- storage/perfschema/CMakeLists.txt | 2 + storage/perfschema/ha_perfschema.cc | 2 +- storage/perfschema/pfs.cc | 2 +- storage/perfschema/pfs_dd_version.h | 7 +- storage/perfschema/pfs_engine_table.cc | 2 + .../perfschema/table_replica_statistics.cc | 149 +++++++++ storage/perfschema/table_replica_statistics.h | 101 ++++++ storage/perfschema/table_threads.cc | 3 +- 58 files changed, 1896 insertions(+), 190 deletions(-) create mode 100644 mysql-test/r/slave_stats_daemon.result create mode 100644 mysql-test/suite/perfschema/r/ddl_replica_statistics.result create mode 100644 mysql-test/suite/perfschema/r/dml_replica_statistics.result create mode 100644 mysql-test/suite/perfschema/t/ddl_replica_statistics.test create mode 100644 mysql-test/suite/perfschema/t/dml_replica_statistics.test create mode 100644 mysql-test/suite/sys_vars/r/write_stats_count_basic.result create mode 100644 mysql-test/suite/sys_vars/r/write_stats_frequency_basic.result create mode 100644 mysql-test/suite/sys_vars/t/write_stats_count_basic.test create mode 100644 mysql-test/suite/sys_vars/t/write_stats_frequency_basic.test create mode 100644 mysql-test/t/slave_stats_daemon.test create mode 100644 sql/slave_stats_daemon.cc create mode 100644 sql/slave_stats_daemon.h create mode 100644 storage/perfschema/table_replica_statistics.cc create mode 100644 storage/perfschema/table_replica_statistics.h diff --git a/client/check/mysqlcheck_core.cc b/client/check/mysqlcheck_core.cc index 3a6bf706b2b1..d1a01cd841df 100644 --- a/client/check/mysqlcheck_core.cc +++ b/client/check/mysqlcheck_core.cc @@ -214,7 +214,7 @@ static int use_db(const string &database) { return 1; if (mysql_get_server_version(sock) >= FIRST_PERFORMANCE_SCHEMA_VERSION && !my_strcasecmp(&my_charset_latin1, database.c_str(), - PERFORMANCE_SCHEMA_DB_NAME)) + PERFORMANCE_SCHEMA_DB_NAME_MACRO)) return 1; if (mysql_select_db(sock, database.c_str())) { DBError(sock, "when selecting the database"); diff --git a/client/client_priv.h b/client/client_priv.h index 41df0ba9b1ff..f5825f28070b 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -225,7 +225,7 @@ enum options_client { /** Name of the performance schema database. */ -#define PERFORMANCE_SCHEMA_DB_NAME "performance_schema" +#define PERFORMANCE_SCHEMA_DB_NAME_MACRO "performance_schema" /** First mysql version supporting the sys schema. diff --git a/client/dump/mysqldump_tool_chain_maker_options.cc b/client/dump/mysqldump_tool_chain_maker_options.cc index eb4e4e1e9833..1b9bebf26b00 100644 --- a/client/dump/mysqldump_tool_chain_maker_options.cc +++ b/client/dump/mysqldump_tool_chain_maker_options.cc @@ -210,7 +210,7 @@ void Mysqldump_tool_chain_maker_options::process_positional_options( m_object_filter.m_databases_excluded.push_back( std::make_pair("", INFORMATION_SCHEMA_DB_NAME)); m_object_filter.m_databases_excluded.push_back( - std::make_pair("", PERFORMANCE_SCHEMA_DB_NAME)); + std::make_pair("", PERFORMANCE_SCHEMA_DB_NAME_MACRO)); m_object_filter.m_databases_excluded.push_back( std::make_pair("", "ndbinfo")); m_object_filter.m_databases_excluded.push_back(std::make_pair("", "sys")); diff --git a/client/mysqldump.cc b/client/mysqldump.cc index de287928f74d..8db67e5710b8 100644 --- a/client/mysqldump.cc +++ b/client/mysqldump.cc @@ -4766,7 +4766,8 @@ static int dump_all_databases() { continue; if (mysql_get_server_version(mysql) >= FIRST_PERFORMANCE_SCHEMA_VERSION && - !my_strcasecmp(&my_charset_latin1, row[0], PERFORMANCE_SCHEMA_DB_NAME)) + !my_strcasecmp(&my_charset_latin1, row[0], + PERFORMANCE_SCHEMA_DB_NAME_MACRO)) continue; if (mysql_get_server_version(mysql) >= FIRST_SYS_SCHEMA_VERSION && @@ -4814,7 +4815,7 @@ static int dump_all_databases() { if (mysql_get_server_version(mysql) >= FIRST_PERFORMANCE_SCHEMA_VERSION && !my_strcasecmp(&my_charset_latin1, row[0], - PERFORMANCE_SCHEMA_DB_NAME)) + PERFORMANCE_SCHEMA_DB_NAME_MACRO)) continue; if (mysql_get_server_version(mysql) >= FIRST_SYS_SCHEMA_VERSION && @@ -5249,7 +5250,8 @@ static int dump_selected_tables(char *db, char **table_names, int tables) { !(mysql_get_server_version(mysql) >= FIRST_INFORMATION_SCHEMA_VERSION && !my_strcasecmp(&my_charset_latin1, db, INFORMATION_SCHEMA_DB_NAME)) && !(mysql_get_server_version(mysql) >= FIRST_PERFORMANCE_SCHEMA_VERSION && - !my_strcasecmp(&my_charset_latin1, db, PERFORMANCE_SCHEMA_DB_NAME))) { + !my_strcasecmp(&my_charset_latin1, db, + PERFORMANCE_SCHEMA_DB_NAME_MACRO))) { if (mysql_real_query(mysql, lock_tables_query.str, (ulong)(lock_tables_query.length - 1))) { if (!opt_force) { diff --git a/include/map_helpers.h b/include/map_helpers.h index 8d5629a5a047..e9e916eb7493 100644 --- a/include/map_helpers.h +++ b/include/map_helpers.h @@ -87,13 +87,24 @@ typename Container::iterator erase_specific_element( template using unique_ptr_with_deleter = std::unique_ptr; +template struct My_free_deleter { + void operator()(void *ptr) const { + reinterpret_cast(ptr)->~T(); + my_free(ptr); + } +}; + +template +struct My_free_deleter { void operator()(void *ptr) const { my_free(ptr); } }; /** std::unique_ptr, but with my_free as deleter. */ template -using unique_ptr_my_free = std::unique_ptr; +using unique_ptr_my_free = std::unique_ptr< + T, My_free_deleter::value && + !std::is_trivially_destructible::value>>; struct Free_deleter { void operator()(void *ptr) const { free(ptr); } diff --git a/include/my_command.h b/include/my_command.h index d5503d78bf6a..22fae26b7e3f 100644 --- a/include/my_command.h +++ b/include/my_command.h @@ -102,7 +102,10 @@ enum enum_server_command { The following are Facebook specific commands. They are put at the top end to avoid conflicting with upstream. */ + COM_TOP_BEGIN = 253, + COM_SEND_REPLICA_STATISTICS = 254, COM_QUERY_ATTRS = 255, + COM_TOP_END = 256, }; #endif /* _mysql_command_h */ diff --git a/include/mysql.h.pp b/include/mysql.h.pp index 60fc89ac3226..11cc1e1fe78f 100644 --- a/include/mysql.h.pp +++ b/include/mysql.h.pp @@ -88,7 +88,10 @@ COM_CLONE, COM_SUBSCRIBE_GROUP_REPLICATION_STREAM, COM_END, + COM_TOP_BEGIN = 253, + COM_SEND_REPLICA_STATISTICS = 254, COM_QUERY_ATTRS = 255, + COM_TOP_END = 256, }; #include "my_compress.h" enum enum_compression_algorithm { diff --git a/include/mysql/plugin_audit.h.pp b/include/mysql/plugin_audit.h.pp index cc9d8ab4a0dc..60763a6c40bd 100644 --- a/include/mysql/plugin_audit.h.pp +++ b/include/mysql/plugin_audit.h.pp @@ -180,7 +180,10 @@ COM_CLONE, COM_SUBSCRIBE_GROUP_REPLICATION_STREAM, COM_END, + COM_TOP_BEGIN = 253, + COM_SEND_REPLICA_STATISTICS = 254, COM_QUERY_ATTRS = 255, + COM_TOP_END = 256, }; #include "my_sqlcommand.h" enum enum_sql_command { diff --git a/mysql-test/include/check-testcase.test b/mysql-test/include/check-testcase.test index 39f7164e5a3b..06bc37f5cf38 100644 --- a/mysql-test/include/check-testcase.test +++ b/mysql-test/include/check-testcase.test @@ -96,6 +96,7 @@ if ($tmp) { --echo Source_public_key_path --echo Get_Source_public_key 0 --echo Network_Namespace + --echo Slave_Lag_Stats_Thread_Running No } if (!$tmp) { diff --git a/mysql-test/r/disabled_replication.result b/mysql-test/r/disabled_replication.result index 3d2dc7d166ef..9bfa668d6da1 100644 --- a/mysql-test/r/disabled_replication.result +++ b/mysql-test/r/disabled_replication.result @@ -1,5 +1,5 @@ SHOW SLAVE STATUS; -Slave_IO_State Master_Host Master_User Master_Port Connect_Retry Master_Log_File Read_Master_Log_Pos Relay_Log_File Relay_Log_Pos Relay_Master_Log_File Slave_IO_Running Slave_SQL_Running Replicate_Do_DB Replicate_Ignore_DB Replicate_Do_Table Replicate_Ignore_Table Replicate_Wild_Do_Table Replicate_Wild_Ignore_Table Last_Errno Last_Symbolic_Errno Last_Error Skip_Counter Exec_Master_Log_Pos Relay_Log_Space Until_Condition Until_Log_File Until_Log_Pos Master_SSL_Allowed Master_SSL_CA_File Master_SSL_CA_Path Master_SSL_Cert Master_SSL_Cipher Master_SSL_Key Seconds_Behind_Master Master_SSL_Verify_Server_Cert Last_IO_Errno Last_IO_Error Last_SQL_Errno Last_SQL_Error Replicate_Ignore_Server_Ids Master_Server_Id Master_UUID Master_Info_File SQL_Delay SQL_Remaining_Delay Slave_SQL_Running_State Master_Retry_Count Master_Bind Last_IO_Error_Timestamp Last_SQL_Error_Timestamp Master_SSL_Crl Master_SSL_Crlpath Retrieved_Gtid_Set Executed_Gtid_Set Auto_Position Replicate_Rewrite_DB Channel_Name Master_TLS_Version Master_public_key_path Get_master_public_key Network_Namespace +Slave_IO_State Master_Host Master_User Master_Port Connect_Retry Master_Log_File Read_Master_Log_Pos Relay_Log_File Relay_Log_Pos Relay_Master_Log_File Slave_IO_Running Slave_SQL_Running Replicate_Do_DB Replicate_Ignore_DB Replicate_Do_Table Replicate_Ignore_Table Replicate_Wild_Do_Table Replicate_Wild_Ignore_Table Last_Errno Last_Symbolic_Errno Last_Error Skip_Counter Exec_Master_Log_Pos Relay_Log_Space Until_Condition Until_Log_File Until_Log_Pos Master_SSL_Allowed Master_SSL_CA_File Master_SSL_CA_Path Master_SSL_Cert Master_SSL_Cipher Master_SSL_Key Seconds_Behind_Master Master_SSL_Verify_Server_Cert Last_IO_Errno Last_IO_Error Last_SQL_Errno Last_SQL_Error Replicate_Ignore_Server_Ids Master_Server_Id Master_UUID Master_Info_File SQL_Delay SQL_Remaining_Delay Slave_SQL_Running_State Master_Retry_Count Master_Bind Last_IO_Error_Timestamp Last_SQL_Error_Timestamp Master_SSL_Crl Master_SSL_Crlpath Retrieved_Gtid_Set Executed_Gtid_Set Auto_Position Replicate_Rewrite_DB Channel_Name Master_TLS_Version Master_public_key_path Get_master_public_key Network_Namespace Slave_Lag_Stats_Thread_Running Warnings: Warning 1287 'SHOW SLAVE STATUS' is deprecated and will be removed in a future release. Please use SHOW REPLICA STATUS instead RESET SLAVE; diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index a7cab41722b5..bee9892da698 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -2519,6 +2519,14 @@ The following options may be given as the first argument: inversion optimization for moving window frames also for floating values. (Defaults to on; use --skip-windowing-use-high-precision to disable.) + --write-stats-count[=#] + Maximum number of most recent data points to be collected + for information_schema.write_statistics & + information_schema.replica_statistics time series. + --write-stats-frequency[=#] + This variable determines the frequency(seconds) at which + write stats and replica lag stats are collected on + primaries --zstd-net-compression-level[=#] Compression level for compressed protocol when zstd library is selected. @@ -2870,7 +2878,7 @@ performance-schema-max-socket-classes 10 performance-schema-max-socket-instances -1 performance-schema-max-sql-text-length 1024 performance-schema-max-stage-classes 175 -performance-schema-max-statement-classes 230 +performance-schema-max-statement-classes 232 performance-schema-max-statement-stack 10 performance-schema-max-table-handles -1 performance-schema-max-table-instances -1 @@ -3232,6 +3240,8 @@ validate-user-plugins TRUE verbose TRUE wait-timeout 28800 windowing-use-high-precision TRUE +write-stats-count 0 +write-stats-frequency 0 zstd-net-compression-level 3 To see what values a running MySQL server is using, type diff --git a/mysql-test/r/slave_stats_daemon.result b/mysql-test/r/slave_stats_daemon.result new file mode 100644 index 000000000000..92152a45f961 --- /dev/null +++ b/mysql-test/r/slave_stats_daemon.result @@ -0,0 +1,197 @@ +include/master-slave.inc +[connection master] +######################################################################################################## +### Case 1: Stats are not sent by default i.e. interval value set to 0 +######################################################################################################## +select @@write_stats_frequency; +@@write_stats_frequency +0 +select @@write_stats_count; +@@write_stats_count +0 +select sleep(2); +sleep(2) +0 +select count(*) = 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +######################################################################################################## +### Case 2: Stats are sent when interval value is set > 0 on slaves +######################################################################################################## +set @@GLOBAL.write_stats_frequency=1; +select @@write_stats_frequency; +@@write_stats_frequency +1 +set @@GLOBAL.write_stats_count=10; +select sleep(2); +sleep(2) +0 +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +######################################################################################################## +### Case 2.1: Dynamically updating write_stats_count updates the number of data points in replica_statistics +######################################################################################################## +set @@GLOBAL.write_stats_frequency=1; +select @@write_stats_frequency; +@@write_stats_frequency +1 +set @@GLOBAL.write_stats_count=1; +select sleep(2); +sleep(2) +0 +select count(*) = 1 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +set @@GLOBAL.write_stats_count=0; +select sleep(2); +sleep(2) +0 +select count(*) = 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +set @@GLOBAL.write_stats_count=2; +select sleep(3); +sleep(3) +0 +select count(*) = 2 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +######################################################################################################## +### Case 2.5: Connection is restored and stats are sent to master after it stops and restarts +######################################################################################################## +set @@GLOBAL.write_stats_frequency=1; +select @@write_stats_frequency; +@@write_stats_frequency +1 +set @@GLOBAL.write_stats_count=10; +select sleep(1); +sleep(1) +0 +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +select sleep(1); +sleep(1) +0 +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +######################################################################################################## +### Case 3: In case of master promotions, slave thread is able to reconnet to new master. +### In this test, I point the slave to an unavailable new master and then re-point it back to original +### master to verify if stats communication resumes. +######################################################################################################## +set @@GLOBAL.write_stats_frequency=1; +Master_User = 'root' +Master_Host = '127.0.0.1' +include/stop_slave.inc +change master to master_user='test'; +start slave; +include/wait_for_slave_io_error.inc [errno=1045] +Master_User = 'test' +Master_Host = '127.0.0.1' +set @@GLOBAL.write_stats_count=10; +select sleep(2); +sleep(2) +0 +select now() - max(timestamp) > 1 as more_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; +more_than_1_sec_old_most_recent_stats +1 +include/stop_slave_sql.inc +change master to master_user='root'; +include/start_slave.inc +Master_User = 'root' +Master_Host = '127.0.0.1' +select sleep(2); +sleep(2) +0 +select now() - max(timestamp) <= 1 as less_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; +less_than_1_sec_old_most_recent_stats +1 +######################################################################################################## +### Case 3.5: Slave should be able to handle reconnections to master. It should reuse the existing +### thread for sending slave stats and should not spawn a new one. +######################################################################################################## +select @@write_stats_frequency; +@@write_stats_frequency +1 +set @@GLOBAL.write_stats_frequency=1; +set @@GLOBAL.write_stats_count=10; +select sleep(2); +sleep(2) +0 +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +select id from information_schema.processlist where command='Binlog Dump' into @id; +kill @id; +select sleep(2); +sleep(2) +0 +select now() - max(timestamp) <= 1 as less_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; +less_than_1_sec_old_most_recent_stats +1 +######################################################################################################## +### Case 3.5: Secondary is not stuck waiting for primary to send OK packet in case something goes wrong +### It should timeout and move on. +######################################################################################################## +select @@write_stats_frequency; +@@write_stats_frequency +1 +set @@GLOBAL.write_stats_frequency=1; +set @@GLOBAL.write_stats_count=10; +select sleep(2); +sleep(2) +0 +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +include/stop_slave.inc +include/start_slave.inc +select sleep(2); +sleep(2) +0 +select now() - max(timestamp) <= 1 as less_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; +less_than_1_sec_old_most_recent_stats +1 +######################################################################################################## +### Case 4: Promote slave to master and master to slave. Old master(new slave) should be able to send +### stats to new master(old slave) +######################################################################################################## +set @@GLOBAL.write_stats_count=10; +set @@GLOBAL.write_stats_frequency=1; +set @@GLOBAL.write_stats_count=10; +set @@GLOBAL.write_stats_frequency=1; +select sleep(2); +sleep(2) +0 +select count(*) from performance_schema.replica_statistics where (now() - timestamp) <= 1; +count(*) +0 +include/stop_slave.inc +reset slave all; +CHANGE MASTER TO MASTER_HOST= 'MASTER_HOST', MASTER_USER= 'root', MASTER_PORT= MASTER_PORT;; +include/start_slave.inc +select sleep(2); +sleep(2) +0 +select sleep(2); +sleep(2) +0 +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +stats_samples_collected +1 +######################################################################################################## +### Cleanup +### Reset the topology, swap master and slaves again +######################################################################################################## +set @@GLOBAL.write_stats_count=0; +set @@GLOBAL.write_stats_frequency=0; +include/stop_slave.inc +reset slave all; +set @@GLOBAL.write_stats_count=0; +set @@GLOBAL.write_stats_frequency=0; +CHANGE MASTER TO MASTER_HOST= 'MASTER_HOST', MASTER_USER= 'root', MASTER_PORT= MASTER_PORT;; +include/start_slave.inc +include/rpl_end.inc diff --git a/mysql-test/suite/perfschema/r/all_tests.result b/mysql-test/suite/perfschema/r/all_tests.result index 94905ffdee17..b1c7acd06b4c 100644 --- a/mysql-test/suite/perfschema/r/all_tests.result +++ b/mysql-test/suite/perfschema/r/all_tests.result @@ -33,6 +33,7 @@ dml_table_statistics_by_table.test idx_binary_log_transaction_compression_stats.test idx_esms_by_all.test idx_log_status.test +idx_replica_statistics.test idx_replication_applier_filters.test idx_replication_applier_global_filters.test idx_replication_asynchronous_connection_failover.test diff --git a/mysql-test/suite/perfschema/r/dd_version_check.result b/mysql-test/suite/perfschema/r/dd_version_check.result index 92b81078c451..54b408d543e6 100644 --- a/mysql-test/suite/perfschema/r/dd_version_check.result +++ b/mysql-test/suite/perfschema/r/dd_version_check.result @@ -1,6 +1,6 @@ "Checking the data dictionary properties ..." SUBSTRING_INDEX(SUBSTRING(properties, LOCATE('PS_VERSION', properties), 30), ';', 1) -PS_VERSION=80028007 +PS_VERSION=80028008 "Checking the performance schema database structure ..." CHECK STATUS -The tables in the performance_schema were last changed in MySQL 8.0.28-007 +The tables in the performance_schema were last changed in MySQL 8.0.28-008 diff --git a/mysql-test/suite/perfschema/r/ddl_replica_statistics.result b/mysql-test/suite/perfschema/r/ddl_replica_statistics.result new file mode 100644 index 000000000000..f98b12283f7f --- /dev/null +++ b/mysql-test/suite/perfschema/r/ddl_replica_statistics.result @@ -0,0 +1,11 @@ +ALTER TABLE performance_schema.replica_statistics +ADD COLUMN foo integer; +ERROR 42000: Access denied for user 'root'@'localhost' to database 'performance_schema' +TRUNCATE TABLE performance_schema.replica_statistics; +ERROR 42000: DROP command denied to user 'root'@'localhost' for table 'replica_statistics' +ALTER TABLE performance_schema.replica_statistics +ADD INDEX test_index(SERVER_ID); +ERROR 42000: Access denied for user 'root'@'localhost' to database 'performance_schema' +CREATE UNIQUE INDEX test_index ON +performance_schema.replica_statistics(SERVER_ID); +ERROR 42000: Access denied for user 'root'@'localhost' to database 'performance_schema' diff --git a/mysql-test/suite/perfschema/r/dml_handler.result b/mysql-test/suite/perfschema/r/dml_handler.result index 049be7a5647f..b42de004b970 100644 --- a/mysql-test/suite/perfschema/r/dml_handler.result +++ b/mysql-test/suite/perfschema/r/dml_handler.result @@ -9,132 +9,135 @@ SELECT COUNT(*) FROM table_list INTO @table_count; # For each table in the performance schema, attempt HANDLER...OPEN, # which should fail with an error 1031, ER_ILLEGAL_HA. -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=113; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=114; HANDLER performance_schema.variables_info OPEN; ERROR HY000: Table storage engine for 'variables_info' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=112; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=113; HANDLER performance_schema.variables_by_thread OPEN; ERROR HY000: Table storage engine for 'variables_by_thread' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=111; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=112; HANDLER performance_schema.users OPEN; ERROR HY000: Table storage engine for 'users' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=110; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=111; HANDLER performance_schema.user_variables_by_thread OPEN; ERROR HY000: Table storage engine for 'user_variables_by_thread' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=109; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=110; HANDLER performance_schema.user_defined_functions OPEN; ERROR HY000: Table storage engine for 'user_defined_functions' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=108; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=109; HANDLER performance_schema.tls_channel_status OPEN; ERROR HY000: Table storage engine for 'tls_channel_status' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=107; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=108; HANDLER performance_schema.threads OPEN; ERROR HY000: Table storage engine for 'threads' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=106; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=107; HANDLER performance_schema.table_statistics_by_table OPEN; ERROR HY000: Table storage engine for 'table_statistics_by_table' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=105; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=106; HANDLER performance_schema.table_lock_waits_summary_by_table OPEN; ERROR HY000: Table storage engine for 'table_lock_waits_summary_by_table' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=104; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=105; HANDLER performance_schema.table_io_waits_summary_by_table OPEN; ERROR HY000: Table storage engine for 'table_io_waits_summary_by_table' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=103; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=104; HANDLER performance_schema.table_io_waits_summary_by_index_usage OPEN; ERROR HY000: Table storage engine for 'table_io_waits_summary_by_index_usage' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=102; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=103; HANDLER performance_schema.table_handles OPEN; ERROR HY000: Table storage engine for 'table_handles' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=101; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=102; HANDLER performance_schema.status_by_user OPEN; ERROR HY000: Table storage engine for 'status_by_user' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=100; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=101; HANDLER performance_schema.status_by_thread OPEN; ERROR HY000: Table storage engine for 'status_by_thread' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=99; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=100; HANDLER performance_schema.status_by_host OPEN; ERROR HY000: Table storage engine for 'status_by_host' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=98; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=99; HANDLER performance_schema.status_by_account OPEN; ERROR HY000: Table storage engine for 'status_by_account' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=97; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=98; HANDLER performance_schema.socket_summary_by_instance OPEN; ERROR HY000: Table storage engine for 'socket_summary_by_instance' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=96; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=97; HANDLER performance_schema.socket_summary_by_event_name OPEN; ERROR HY000: Table storage engine for 'socket_summary_by_event_name' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=95; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=96; HANDLER performance_schema.socket_instances OPEN; ERROR HY000: Table storage engine for 'socket_instances' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=94; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=95; HANDLER performance_schema.setup_threads OPEN; ERROR HY000: Table storage engine for 'setup_threads' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=93; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=94; HANDLER performance_schema.setup_objects OPEN; ERROR HY000: Table storage engine for 'setup_objects' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=92; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=93; HANDLER performance_schema.setup_instruments OPEN; ERROR HY000: Table storage engine for 'setup_instruments' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=91; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=92; HANDLER performance_schema.setup_consumers OPEN; ERROR HY000: Table storage engine for 'setup_consumers' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=90; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=91; HANDLER performance_schema.setup_actors OPEN; ERROR HY000: Table storage engine for 'setup_actors' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=89; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=90; HANDLER performance_schema.session_variables OPEN; ERROR HY000: Table storage engine for 'session_variables' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=88; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=89; HANDLER performance_schema.session_status OPEN; ERROR HY000: Table storage engine for 'session_status' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=87; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=88; HANDLER performance_schema.session_query_attrs OPEN; ERROR HY000: Table storage engine for 'session_query_attrs' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=86; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=87; HANDLER performance_schema.session_connect_attrs OPEN; ERROR HY000: Table storage engine for 'session_connect_attrs' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=85; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=86; HANDLER performance_schema.session_account_connect_attrs OPEN; ERROR HY000: Table storage engine for 'session_account_connect_attrs' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=84; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=85; HANDLER performance_schema.rwlock_instances OPEN; ERROR HY000: Table storage engine for 'rwlock_instances' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=83; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=84; HANDLER performance_schema.replication_group_members OPEN; ERROR HY000: Table storage engine for 'replication_group_members' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=82; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=83; HANDLER performance_schema.replication_group_member_stats OPEN; ERROR HY000: Table storage engine for 'replication_group_member_stats' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=81; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=82; HANDLER performance_schema.replication_connection_status OPEN; ERROR HY000: Table storage engine for 'replication_connection_status' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=80; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=81; HANDLER performance_schema.replication_connection_configuration OPEN; ERROR HY000: Table storage engine for 'replication_connection_configuration' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=79; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=80; HANDLER performance_schema.replication_asynchronous_connection_failover_managed OPEN; ERROR HY000: Table storage engine for 'replication_asynchronous_connection_failover_managed' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=78; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=79; HANDLER performance_schema.replication_asynchronous_connection_failover OPEN; ERROR HY000: Table storage engine for 'replication_asynchronous_connection_failover' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=77; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=78; HANDLER performance_schema.replication_applier_status_by_worker OPEN; ERROR HY000: Table storage engine for 'replication_applier_status_by_worker' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=76; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=77; HANDLER performance_schema.replication_applier_status_by_coordinator OPEN; ERROR HY000: Table storage engine for 'replication_applier_status_by_coordinator' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=75; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=76; HANDLER performance_schema.replication_applier_status OPEN; ERROR HY000: Table storage engine for 'replication_applier_status' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=74; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=75; HANDLER performance_schema.replication_applier_global_filters OPEN; ERROR HY000: Table storage engine for 'replication_applier_global_filters' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=73; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=74; HANDLER performance_schema.replication_applier_filters OPEN; ERROR HY000: Table storage engine for 'replication_applier_filters' doesn't have this option -SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=72; +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=73; HANDLER performance_schema.replication_applier_configuration OPEN; ERROR HY000: Table storage engine for 'replication_applier_configuration' doesn't have this option +SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=72; +HANDLER performance_schema.replica_statistics OPEN; +ERROR HY000: Table storage engine for 'replica_statistics' doesn't have this option SELECT TABLE_NAME INTO @table_name FROM table_list WHERE id=71; HANDLER performance_schema.processlist OPEN; ERROR HY000: Table storage engine for 'processlist' doesn't have this option diff --git a/mysql-test/suite/perfschema/r/dml_replica_statistics.result b/mysql-test/suite/perfschema/r/dml_replica_statistics.result new file mode 100644 index 000000000000..0a809a50c57e --- /dev/null +++ b/mysql-test/suite/perfschema/r/dml_replica_statistics.result @@ -0,0 +1,19 @@ +SELECT * FROM performance_schema.replica_statistics +LIMIT 1; +INSERT INTO performance_schema.replica_statistics +VALUES(123, NOW(), 10); +ERROR 42000: INSERT command denied to user 'root'@'localhost' for table 'replica_statistics' +UPDATE performance_schema.replica_statistics +SET SERVER_ID=123 WHERE SERVER_ID=2; +ERROR 42000: UPDATE command denied to user 'root'@'localhost' for table 'replica_statistics' +DELETE FROM performance_schema.replica_statistics +WHERE SERVER_ID=2; +ERROR 42000: DELETE command denied to user 'root'@'localhost' for table 'replica_statistics' +DELETE FROM performance_schema.replica_statistics; +ERROR 42000: DELETE command denied to user 'root'@'localhost' for table 'replica_statistics' +LOCK TABLES performance_schema.replica_statistics READ; +ERROR 42000: SELECT, LOCK TABLES command denied to user 'root'@'localhost' for table 'replica_statistics' +UNLOCK TABLES; +LOCK TABLES performance_schema.replica_statistics WRITE; +ERROR 42000: SELECT, LOCK TABLES command denied to user 'root'@'localhost' for table 'replica_statistics' +UNLOCK TABLES; diff --git a/mysql-test/suite/perfschema/r/dml_setup_instruments.result b/mysql-test/suite/perfschema/r/dml_setup_instruments.result index 78baf44891d4..c33f074406cd 100644 --- a/mysql-test/suite/perfschema/r/dml_setup_instruments.result +++ b/mysql-test/suite/perfschema/r/dml_setup_instruments.result @@ -48,9 +48,9 @@ wait/synch/cond/sql/COND_flush_thread_cache YES YES singleton 0 NULL wait/synch/cond/sql/COND_manager YES YES singleton 0 NULL wait/synch/cond/sql/COND_queue_state YES YES singleton 0 NULL wait/synch/cond/sql/COND_server_started YES YES singleton 0 NULL +wait/synch/cond/sql/COND_slave_stats_daemon YES YES singleton 0 NULL wait/synch/cond/sql/COND_thd_list YES YES 0 NULL wait/synch/cond/sql/COND_thr_lock YES YES 0 NULL -wait/synch/cond/sql/COND_thread_cache YES YES singleton 0 NULL select * from performance_schema.setup_instruments where name='Wait'; select * from performance_schema.setup_instruments diff --git a/mysql-test/suite/perfschema/r/information_schema.result b/mysql-test/suite/perfschema/r/information_schema.result index e715fd035867..554ea317aff7 100644 --- a/mysql-test/suite/perfschema/r/information_schema.result +++ b/mysql-test/suite/perfschema/r/information_schema.result @@ -73,6 +73,7 @@ performance_schema performance_timers def performance_schema persisted_variables def performance_schema prepared_statements_instances def performance_schema processlist def +performance_schema replica_statistics def performance_schema replication_applier_configuration def performance_schema replication_applier_filters def performance_schema replication_applier_global_filters def @@ -190,6 +191,7 @@ performance_timers BASE TABLE PERFORMANCE_SCHEMA persisted_variables BASE TABLE PERFORMANCE_SCHEMA prepared_statements_instances BASE TABLE PERFORMANCE_SCHEMA processlist BASE TABLE PERFORMANCE_SCHEMA +replica_statistics BASE TABLE PERFORMANCE_SCHEMA replication_applier_configuration BASE TABLE PERFORMANCE_SCHEMA replication_applier_filters BASE TABLE PERFORMANCE_SCHEMA replication_applier_global_filters BASE TABLE PERFORMANCE_SCHEMA @@ -307,6 +309,7 @@ performance_timers 10 Fixed persisted_variables 10 Dynamic prepared_statements_instances 10 Dynamic processlist 10 Dynamic +replica_statistics 10 Fixed replication_applier_configuration 10 Dynamic replication_applier_filters 10 Dynamic replication_applier_global_filters 10 Dynamic @@ -422,6 +425,7 @@ objects_summary_global_by_type 0 performance_timers 0 prepared_statements_instances 0 processlist 0 +replica_statistics 0 replication_applier_configuration 0 replication_applier_filters 0 replication_applier_global_filters 0 @@ -547,6 +551,7 @@ performance_timers 0 0 persisted_variables 0 0 prepared_statements_instances 0 0 processlist 0 0 +replica_statistics 0 0 replication_applier_configuration 0 0 replication_applier_filters 0 0 replication_applier_global_filters 0 0 @@ -664,6 +669,7 @@ performance_timers 0 0 NULL persisted_variables 0 0 NULL prepared_statements_instances 0 0 NULL processlist 0 0 NULL +replica_statistics 0 0 NULL replication_applier_configuration 0 0 NULL replication_applier_filters 0 0 NULL replication_applier_global_filters 0 0 NULL @@ -781,6 +787,7 @@ performance_timers NULL NULL NULL persisted_variables NULL NULL NULL prepared_statements_instances NULL NULL NULL processlist NULL NULL NULL +replica_statistics NULL NULL NULL replication_applier_configuration NULL NULL NULL replication_applier_filters NULL NULL NULL replication_applier_global_filters NULL NULL NULL @@ -898,6 +905,7 @@ performance_timers utf8mb4_0900_ai_ci NULL persisted_variables utf8mb4_0900_ai_ci NULL prepared_statements_instances utf8mb4_0900_ai_ci NULL processlist utf8mb4_0900_ai_ci NULL +replica_statistics utf8mb4_0900_ai_ci NULL replication_applier_configuration utf8mb4_0900_ai_ci NULL replication_applier_filters utf8mb4_0900_ai_ci NULL replication_applier_global_filters utf8mb4_0900_ai_ci NULL @@ -1015,6 +1023,7 @@ performance_timers persisted_variables prepared_statements_instances processlist +replica_statistics replication_applier_configuration replication_applier_filters replication_applier_global_filters @@ -1132,6 +1141,7 @@ performance_timers persisted_variables prepared_statements_instances processlist +replica_statistics replication_applier_configuration replication_applier_filters replication_applier_global_filters diff --git a/mysql-test/suite/perfschema/r/schema.result b/mysql-test/suite/perfschema/r/schema.result index a86b5be29e50..130ab80a7c24 100644 --- a/mysql-test/suite/perfschema/r/schema.result +++ b/mysql-test/suite/perfschema/r/schema.result @@ -78,6 +78,7 @@ performance_timers persisted_variables prepared_statements_instances processlist +replica_statistics replication_applier_configuration replication_applier_filters replication_applier_global_filters diff --git a/mysql-test/suite/perfschema/r/table_schema.result b/mysql-test/suite/perfschema/r/table_schema.result index e8f4f19ca975..7e900904ea8f 100644 --- a/mysql-test/suite/perfschema/r/table_schema.result +++ b/mysql-test/suite/perfschema/r/table_schema.result @@ -1259,6 +1259,9 @@ def performance_schema replication_group_member_stats COUNT_TRANSACTIONS_REMOTE_ def performance_schema replication_group_member_stats COUNT_TRANSACTIONS_REMOTE_APPLIED 11 NULL NO bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned select,insert,update,references NULL def performance_schema replication_group_member_stats COUNT_TRANSACTIONS_LOCAL_PROPOSED 12 NULL NO bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned select,insert,update,references NULL def performance_schema replication_group_member_stats COUNT_TRANSACTIONS_LOCAL_ROLLBACK 13 NULL NO bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned select,insert,update,references NULL +def performance_schema replica_statistics SERVER_ID 1 NULL NO bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned select,insert,update,references NULL +def performance_schema replica_statistics TIMESTAMP 2 0000-00-00 00:00:00 NO timestamp NULL NULL NULL NULL 0 NULL NULL timestamp select,insert,update,references NULL +def performance_schema replica_statistics MILLI_SEC_BEHIND_MASTER 3 NULL NO bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned select,insert,update,references NULL def performance_schema rwlock_instances NAME 1 NULL NO varchar 128 512 NULL NULL NULL utf8mb4 utf8mb4_0900_ai_ci varchar(128) MUL select,insert,update,references NULL def performance_schema rwlock_instances OBJECT_INSTANCE_BEGIN 2 NULL NO bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned PRI select,insert,update,references NULL def performance_schema rwlock_instances WRITE_LOCKED_BY_THREAD_ID 3 NULL YES bigint NULL NULL 20 0 NULL NULL NULL bigint unsigned MUL select,insert,update,references NULL diff --git a/mysql-test/suite/perfschema/t/dd_version_check.test b/mysql-test/suite/perfschema/t/dd_version_check.test index 12b9bc04dfd8..a214b34b8728 100644 --- a/mysql-test/suite/perfschema/t/dd_version_check.test +++ b/mysql-test/suite/perfschema/t/dd_version_check.test @@ -133,6 +133,10 @@ insert into test.pfs_published_schema values("MySQL 8.0.28-007", "9ae079063867a00091801219f7f0e297a6a826bb0959302dc3282a6d7e2bdff2"); +insert into test.pfs_published_schema + values("MySQL 8.0.28-008", + "fded54d7b4967d2b78a7dbfd0e812f9df6eb923ce606f3e49e8839906bf67bca"); + create table test.pfs_check_table (id int NOT NULL AUTO_INCREMENT, t text NOT NULL, diff --git a/mysql-test/suite/perfschema/t/ddl_replica_statistics.test b/mysql-test/suite/perfschema/t/ddl_replica_statistics.test new file mode 100644 index 000000000000..0ce7ef06d3a6 --- /dev/null +++ b/mysql-test/suite/perfschema/t/ddl_replica_statistics.test @@ -0,0 +1,21 @@ +# ==== Purpose ==== +# +# Tests for PERFORMANCE_SCHEMA +# +# This test verifies the correct behaviour in case of invalid +# usage of DDLs on the table replica_statistics + +-- error ER_DBACCESS_DENIED_ERROR +ALTER TABLE performance_schema.replica_statistics + ADD COLUMN foo integer; + +-- error ER_TABLEACCESS_DENIED_ERROR +TRUNCATE TABLE performance_schema.replica_statistics; + +-- error ER_DBACCESS_DENIED_ERROR +ALTER TABLE performance_schema.replica_statistics + ADD INDEX test_index(SERVER_ID); + +-- error ER_DBACCESS_DENIED_ERROR +CREATE UNIQUE INDEX test_index ON + performance_schema.replica_statistics(SERVER_ID); diff --git a/mysql-test/suite/perfschema/t/dml_replica_statistics.test b/mysql-test/suite/perfschema/t/dml_replica_statistics.test new file mode 100644 index 000000000000..cc7eef498011 --- /dev/null +++ b/mysql-test/suite/perfschema/t/dml_replica_statistics.test @@ -0,0 +1,35 @@ +# ==== Purpose ==== +# +# Tests for PERFORMANCE_SCHEMA +# +# This test verifies the correct behaviour in case of invalid +# usage of DMLs on the table replica_statistics. +# + +--disable_result_log +SELECT * FROM performance_schema.replica_statistics + LIMIT 1; +--enable_result_log + +--error ER_TABLEACCESS_DENIED_ERROR +INSERT INTO performance_schema.replica_statistics + VALUES(123, NOW(), 10); + +--error ER_TABLEACCESS_DENIED_ERROR +UPDATE performance_schema.replica_statistics + SET SERVER_ID=123 WHERE SERVER_ID=2; + +--error ER_TABLEACCESS_DENIED_ERROR +DELETE FROM performance_schema.replica_statistics + WHERE SERVER_ID=2; + +--error ER_TABLEACCESS_DENIED_ERROR +DELETE FROM performance_schema.replica_statistics; + +-- error ER_TABLEACCESS_DENIED_ERROR +LOCK TABLES performance_schema.replica_statistics READ; +UNLOCK TABLES; + +-- error ER_TABLEACCESS_DENIED_ERROR +LOCK TABLES performance_schema.replica_statistics WRITE; +UNLOCK TABLES; diff --git a/mysql-test/suite/sys_vars/r/write_stats_count_basic.result b/mysql-test/suite/sys_vars/r/write_stats_count_basic.result new file mode 100644 index 000000000000..6ec8a6a3b1ad --- /dev/null +++ b/mysql-test/suite/sys_vars/r/write_stats_count_basic.result @@ -0,0 +1,27 @@ +Default value of write_stats_count is 0 +SELECT @@global.write_stats_count; +@@global.write_stats_count +0 +SELECT @@session.write_stats_count; +ERROR HY000: Variable 'write_stats_count' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +write_stats_count is a dynamic variable (change to 1) +set @@global.write_stats_count = 1; +SELECT @@global.write_stats_count; +@@global.write_stats_count +1 +restore the default value +SET @@global.write_stats_count = 0; +SELECT @@global.write_stats_count; +@@global.write_stats_count +0 +restart the server with non default value (1) +# restart: --write_stats_count=1 +SELECT @@global.write_stats_count; +@@global.write_stats_count +1 +restart the server with the default value (0) +# restart: +SELECT @@global.write_stats_count; +@@global.write_stats_count +0 diff --git a/mysql-test/suite/sys_vars/r/write_stats_frequency_basic.result b/mysql-test/suite/sys_vars/r/write_stats_frequency_basic.result new file mode 100644 index 000000000000..8d0762d304a3 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/write_stats_frequency_basic.result @@ -0,0 +1,27 @@ +Default value of write_stats_frequency is 0 +SELECT @@global.write_stats_frequency; +@@global.write_stats_frequency +0 +SELECT @@session.write_stats_frequency; +ERROR HY000: Variable 'write_stats_frequency' is a GLOBAL variable +Expected error 'Variable is a GLOBAL variable' +write_stats_frequency is a dynamic variable (change to 1) +set @@global.write_stats_frequency = 1; +SELECT @@global.write_stats_frequency; +@@global.write_stats_frequency +1 +restore the default value +SET @@global.write_stats_frequency = 0; +SELECT @@global.write_stats_frequency; +@@global.write_stats_frequency +0 +restart the server with non default value (1) +# restart: --write_stats_frequency=1 +SELECT @@global.write_stats_frequency; +@@global.write_stats_frequency +1 +restart the server with the default value (0) +# restart: +SELECT @@global.write_stats_frequency; +@@global.write_stats_frequency +0 diff --git a/mysql-test/suite/sys_vars/t/write_stats_count_basic.test b/mysql-test/suite/sys_vars/t/write_stats_count_basic.test new file mode 100644 index 000000000000..b5d70c866ae6 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/write_stats_count_basic.test @@ -0,0 +1,44 @@ +-- source include/load_sysvars.inc + +#### +# Verify default value is 0 +#### +--echo Default value of write_stats_count is 0 +SELECT @@global.write_stats_count; + +#### +# Verify that this is not a session variable +#### +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.write_stats_count; +--echo Expected error 'Variable is a GLOBAL variable' + +#### +## Verify that the variable is dynamic +#### +--echo write_stats_count is a dynamic variable (change to 1) +set @@global.write_stats_count = 1; +SELECT @@global.write_stats_count; + +#### +## Restore the default value +#### +--echo restore the default value +SET @@global.write_stats_count = 0; +SELECT @@global.write_stats_count; + +#### +## Restart the server with a non default value of the variable +#### +--echo restart the server with non default value (1) +--let $restart_parameters = restart: --write_stats_count=1 +--source include/restart_mysqld.inc + +SELECT @@global.write_stats_count; + +--echo restart the server with the default value (0) +--let $restart_parameters = restart: +--source include/restart_mysqld.inc + +# check value is default (0) +SELECT @@global.write_stats_count; diff --git a/mysql-test/suite/sys_vars/t/write_stats_frequency_basic.test b/mysql-test/suite/sys_vars/t/write_stats_frequency_basic.test new file mode 100644 index 000000000000..f119dc3e2bc2 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/write_stats_frequency_basic.test @@ -0,0 +1,44 @@ +-- source include/load_sysvars.inc + +#### +# Verify default value is 0 +#### +--echo Default value of write_stats_frequency is 0 +SELECT @@global.write_stats_frequency; + +#### +# Verify that this is not a session variable +#### +--Error ER_INCORRECT_GLOBAL_LOCAL_VAR +SELECT @@session.write_stats_frequency; +--echo Expected error 'Variable is a GLOBAL variable' + +#### +## Verify that the variable is dynamic +#### +--echo write_stats_frequency is a dynamic variable (change to 1) +set @@global.write_stats_frequency = 1; +SELECT @@global.write_stats_frequency; + +#### +## Restore the default value +#### +--echo restore the default value +SET @@global.write_stats_frequency = 0; +SELECT @@global.write_stats_frequency; + +#### +## Restart the server with a non default value of the variable +#### +--echo restart the server with non default value (1) +--let $restart_parameters = restart: --write_stats_frequency=1 +--source include/restart_mysqld.inc + +SELECT @@global.write_stats_frequency; + +--echo restart the server with the default value (0) +--let $restart_parameters = restart: +--source include/restart_mysqld.inc + +# check value is default (0) +SELECT @@global.write_stats_frequency; diff --git a/mysql-test/t/slave_stats_daemon.test b/mysql-test/t/slave_stats_daemon.test new file mode 100644 index 000000000000..9b17e823a0c4 --- /dev/null +++ b/mysql-test/t/slave_stats_daemon.test @@ -0,0 +1,223 @@ +# +# Test if slaves are able to send lag statistics to master every 15 seconds. +# +-- disable_warnings +-- source include/master-slave.inc + +--echo ######################################################################################################## +--echo ### Case 1: Stats are not sent by default i.e. interval value set to 0 +--echo ######################################################################################################## +connection slave; +select @@write_stats_frequency; +select @@write_stats_count; + +connection master; +select sleep(2); +select count(*) = 0 as stats_samples_collected from performance_schema.replica_statistics; + + +--echo ######################################################################################################## +--echo ### Case 2: Stats are sent when interval value is set > 0 on slaves +--echo ######################################################################################################## +connection slave; +set @@GLOBAL.write_stats_frequency=1; +select @@write_stats_frequency; + +connection master; +set @@GLOBAL.write_stats_count=10; +select sleep(2); +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; + +--echo ######################################################################################################## +--echo ### Case 2.1: Dynamically updating write_stats_count updates the number of data points in replica_statistics +--echo ######################################################################################################## +connection slave; +set @@GLOBAL.write_stats_frequency=1; +select @@write_stats_frequency; + +connection master; +set @@GLOBAL.write_stats_count=1; +select sleep(2); +select count(*) = 1 as stats_samples_collected from performance_schema.replica_statistics; + +set @@GLOBAL.write_stats_count=0; +select sleep(2); +select count(*) = 0 as stats_samples_collected from performance_schema.replica_statistics; + +set @@GLOBAL.write_stats_count=2; +select sleep(3); +select count(*) = 2 as stats_samples_collected from performance_schema.replica_statistics; + +--echo ######################################################################################################## +--echo ### Case 2.5: Connection is restored and stats are sent to master after it stops and restarts +--echo ######################################################################################################## +connection slave; +set @@GLOBAL.write_stats_frequency=1; +select @@write_stats_frequency; + +connection master; +set @@GLOBAL.write_stats_count=10; +select sleep(1); +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; +let $rpl_server_number= 1 +-- source include/rpl_restart_server.inc + +connection master; +select sleep(1); +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; + + +--echo ######################################################################################################## +--echo ### Case 3: In case of master promotions, slave thread is able to reconnet to new master. +--echo ### In this test, I point the slave to an unavailable new master and then re-point it back to original +--echo ### master to verify if stats communication resumes. +--echo ######################################################################################################## +connection slave; +set @@GLOBAL.write_stats_frequency=1; +let $status_items= Master_User, Master_Host; +-- source include/show_slave_status.inc + +-- source include/stop_slave.inc +change master to master_user='test'; +start slave; #instead of start_slave.inc to avoid hitting timeout case as io thread won't be able to start +let $slave_io_errno= 1045; +-- source include/wait_for_slave_io_error.inc +-- source include/show_slave_status.inc + +connection master; +set @@GLOBAL.write_stats_count=10; +select sleep(2); + +# No stats should have been received in last 2 seconds +select now() - max(timestamp) > 1 as more_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; + +connection slave; +--source include/stop_slave_sql.inc +change master to master_user='root'; +-- source include/start_slave.inc +-- source include/show_slave_status.inc + +connection master; +select sleep(2); + +# Master should be receiving stats every second and it's most recent stat should not be more than 1 sec old +select now() - max(timestamp) <= 1 as less_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; + +--echo ######################################################################################################## +--echo ### Case 3.5: Slave should be able to handle reconnections to master. It should reuse the existing +--echo ### thread for sending slave stats and should not spawn a new one. +--echo ######################################################################################################## +connection slave; +select @@write_stats_frequency; + +set @@GLOBAL.write_stats_frequency=1; + +connection master; +set @@GLOBAL.write_stats_count=10; +select sleep(2); +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; + +select id from information_schema.processlist where command='Binlog Dump' into @id; +kill @id; # to stimulate reconnection by slave w/o timeout +save_master_pos; + +connection slave; +sync_with_master; + +connection master; +select sleep(2); + +# Master should be receiving stats every second and it's most recent stat should not be more than 1 sec old +select now() - max(timestamp) <= 1 as less_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; + +--echo ######################################################################################################## +--echo ### Case 3.5: Secondary is not stuck waiting for primary to send OK packet in case something goes wrong +--echo ### It should timeout and move on. +--echo ######################################################################################################## +connection slave; +select @@write_stats_frequency; + +set @@GLOBAL.write_stats_frequency=1; + +connection master; +set @@GLOBAL.write_stats_count=10; +select sleep(2); +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; + +let $master_pid_file=`SELECT @@pid_file;`; +exec kill -19 `head -1 $master_pid_file`; + +connection slave; +-- source include/stop_slave.inc ## should not block execution + +## continue master process +exec kill -18 `head -1 $master_pid_file`; +-- source include/start_slave.inc + +connection master; +select sleep(2); + +# Master should be receiving stats every second and it's most recent stat should not be more than 1 sec old +select now() - max(timestamp) <= 1 as less_than_1_sec_old_most_recent_stats from performance_schema.replica_statistics; + +--echo ######################################################################################################## +--echo ### Case 4: Promote slave to master and master to slave. Old master(new slave) should be able to send +--echo ### stats to new master(old slave) +--echo ######################################################################################################## +connection master; +set @@GLOBAL.write_stats_count=10; +set @@GLOBAL.write_stats_frequency=1; + +# get slave port and store it for later use +--let $old_slave_host= query_get_value(SHOW SLAVE HOSTS, Host, 1) +--let $old_slave_port= query_get_value(SHOW SLAVE HOSTS, Port, 1) + +connection slave; +set @@GLOBAL.write_stats_count=10; +set @@GLOBAL.write_stats_frequency=1; + +# get master port and store it for later use during cleanup +--let $old_master_host= query_get_value(SHOW SLAVE STATUS, Master_Host, 1) +--let $old_master_port= query_get_value(SHOW SLAVE STATUS, Master_Port, 1) + +select sleep(2); +# Slave should not have collected any stats recently +select count(*) from performance_schema.replica_statistics where (now() - timestamp) <= 1; + +# promote slave to be the new master +-- source include/stop_slave.inc +reset slave all; + +# Update old master to be a slave and point it to the new master +connection master; +--replace_result $old_slave_host MASTER_HOST $old_slave_port MASTER_PORT +--eval CHANGE MASTER TO MASTER_HOST= '$old_slave_host', MASTER_USER= 'root', MASTER_PORT= $old_slave_port; +-- source include/start_slave.inc +select sleep(2); + +#connect to new master and verify that it has received slave statistics +connection slave; +select sleep(2); +select count(*) > 0 as stats_samples_collected from performance_schema.replica_statistics; + +--echo ######################################################################################################## +--echo ### Cleanup +--echo ### Reset the topology, swap master and slaves again +--echo ######################################################################################################## +# reset original master +connection master; +set @@GLOBAL.write_stats_count=0; +set @@GLOBAL.write_stats_frequency=0; +-- source include/stop_slave.inc +reset slave all; + +# reset original slave +connection slave; +set @@GLOBAL.write_stats_count=0; +set @@GLOBAL.write_stats_frequency=0; +--replace_result $old_master_host MASTER_HOST $old_master_port MASTER_PORT +--eval CHANGE MASTER TO MASTER_HOST= '$old_master_host', MASTER_USER= 'root', MASTER_PORT= $old_master_port; +-- source include/start_slave.inc + +-- enable_warnings +-- source include/rpl_end.inc diff --git a/sql-common/client.cc b/sql-common/client.cc index 63267ddba6d7..7d14db429b7c 100644 --- a/sql-common/client.cc +++ b/sql-common/client.cc @@ -1488,6 +1488,7 @@ bool cli_advanced_command(MYSQL *mysql, enum enum_server_command command, case COM_STMT_SEND_LONG_DATA: case COM_STMT_CLOSE: case COM_REGISTER_SLAVE: + case COM_SEND_REPLICA_STATISTICS: case COM_QUIT: break; @@ -1598,7 +1599,7 @@ net_async_status cli_advanced_command_nonblocking( can happen if a client sends a query but does not reap the result before attempting to close the connection. */ - assert(command <= COM_END || command == COM_QUERY_ATTRS); + assert(command <= COM_END || command > COM_TOP_BEGIN); net_clear(&mysql->net, false); net_async->async_send_command_status = NET_ASYNC_SEND_COMMAND_WRITE_COMMAND; } diff --git a/sql-common/net_serv.cc b/sql-common/net_serv.cc index cb3439ec1c46..1db032a4e316 100644 --- a/sql-common/net_serv.cc +++ b/sql-common/net_serv.cc @@ -539,7 +539,7 @@ static int begin_packet_write_state(NET *net, uchar command, } NET_ASYNC *net_async = NET_ASYNC_DATA(net); size_t total_len = packet_len + prefix_len; - bool include_command = (command != COM_END); + bool include_command = (command < COM_END) || (command > COM_TOP_BEGIN); if (include_command) { ++total_len; } diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 9c362fa814ea..f7bce816f530 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -384,6 +384,7 @@ SET(SQL_SHARED_SOURCES filesort.cc filesort_utils.cc aggregate_check.cc + slave_stats_daemon.cc gstream.cc handler.cc histograms/equi_height.cc diff --git a/sql/json_dom.cc b/sql/json_dom.cc index fab48034901b..a53978ddf0e2 100644 --- a/sql/json_dom.cc +++ b/sql/json_dom.cc @@ -63,6 +63,7 @@ #include "sql/current_thd.h" // current_thd #include "sql/derror.h" // ER_THD #include "sql/field.h" +#include "sql/handler.h" #include "sql/json_path.h" #include "sql/json_syntax_check.h" #include "sql/psi_memory_key.h" // key_memory_JSON diff --git a/sql/log.cc b/sql/log.cc index 5873dfb2db69..385ebfab0ce8 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -2058,7 +2058,7 @@ bool write_log_to_socket(int sockfd, THD *thd, ulonglong end_utime) { (unsigned int)thd->start_time.tv_sec); // query_length == 0 also appears to mean that this is a command if ((!thd->query().str || !thd->query().length) && len < buf_sz) { - if (thd->get_command() < COM_END) + if (thd->get_command() < COM_END || thd->get_command() > COM_TOP_BEGIN) len += snprintf(buf + len, buf_sz - len, "# administrator command: %s\n", Command_names::str_global(thd->get_command()).c_str()); else diff --git a/sql/mysqld.cc b/sql/mysqld.cc index db3b01e05338..baae7e815fc4 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1054,6 +1054,7 @@ PSI_file_key key_file_binlog_index_cache; #ifdef HAVE_PSI_INTERFACE static PSI_mutex_key key_LOCK_status; static PSI_mutex_key key_LOCK_manager; +static PSI_mutex_key key_LOCK_slave_stats_daemon; static PSI_mutex_key key_LOCK_crypt; static PSI_mutex_key key_LOCK_user_conn; static PSI_mutex_key key_LOCK_global_system_variables; @@ -1095,6 +1096,7 @@ static PSI_cond_key key_BINLOG_COND_flush_queue; static PSI_cond_key key_BINLOG_update_cond; static PSI_cond_key key_BINLOG_prep_xids_cond; static PSI_cond_key key_COND_manager; +static PSI_cond_key key_COND_slave_stats_daemon; static PSI_cond_key key_COND_compress_gtid_table; static PSI_thread_key key_thread_signal_hand; static PSI_thread_key key_thread_main; @@ -1394,6 +1396,9 @@ uint32 gtid_executed_compression_period = 0; bool opt_log_unsafe_statements; bool opt_log_global_var_changes; bool is_slave = false; +/* Counter to count the number of slave_stats_daemon threads created. Should be + * at most 1. */ +std::atomic slave_stats_daemon_thread_counter(0); bool read_only_slave; bool flush_only_old_table_cache_entries = false; @@ -1476,6 +1481,12 @@ ulonglong global_conn_mem_counter = 0; bool opt_group_replication_plugin_hooks = false; bool opt_core_file = false; bool skip_core_dump_on_error = false; +/* Controls num most recent data points to collect for + * information_schema.write_statistics */ +uint write_stats_count; +/* Controls the frequency(seconds) at which write stats and replica lag stats + * are collected*/ +ulong write_stats_frequency; bool slave_high_priority_ddl = false; double slave_high_priority_lock_wait_timeout_double = 1.0; ulonglong slave_high_priority_lock_wait_timeout_nsec = 1.0; @@ -2882,6 +2893,7 @@ static void clean_up_mutexes() { mysql_mutex_destroy(&LOCK_log_throttle_ddl); mysql_mutex_destroy(&LOCK_status); mysql_mutex_destroy(&LOCK_manager); + mysql_mutex_destroy(&LOCK_slave_stats_daemon); mysql_mutex_destroy(&LOCK_crypt); mysql_mutex_destroy(&LOCK_user_conn); mysql_rwlock_destroy(&LOCK_sys_init_connect); @@ -2907,6 +2919,7 @@ static void clean_up_mutexes() { mysql_mutex_destroy(&LOCK_password_history); mysql_mutex_destroy(&LOCK_password_reuse_interval); mysql_cond_destroy(&COND_manager); + mysql_cond_destroy(&COND_slave_stats_daemon); #ifdef _WIN32 mysql_cond_destroy(&COND_handler_count); mysql_mutex_destroy(&LOCK_handler_count); @@ -5036,7 +5049,7 @@ static void init_sql_statement_names() { #ifdef HAVE_PSI_STATEMENT_INTERFACE PSI_statement_info sql_statement_info[(uint)SQLCOM_END + 1]; -PSI_statement_info com_statement_info[(uint)COM_END + 1]; +PSI_statement_info com_statement_info[(uint)COM_TOP_END]; /** Initialize the command names array. @@ -5066,7 +5079,7 @@ static void init_sql_statement_info() { static void init_com_statement_info() { uint index; - for (index = 0; index < (uint)COM_END + 1; index++) { + for (index = 0; index < array_elements(com_statement_info); index++) { com_statement_info[index].m_name = Command_names::str_notranslate(index).c_str(); com_statement_info[index].m_flags = 0; @@ -5777,6 +5790,8 @@ int init_common_variables() { static int init_thread_environment() { mysql_mutex_init(key_LOCK_status, &LOCK_status, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_manager, &LOCK_manager, MY_MUTEX_INIT_FAST); + mysql_mutex_init(key_LOCK_slave_stats_daemon, &LOCK_slave_stats_daemon, + MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_crypt, &LOCK_crypt, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_user_conn, &LOCK_user_conn, MY_MUTEX_INIT_FAST); mysql_mutex_init(key_LOCK_global_system_variables, @@ -5820,6 +5835,7 @@ static int init_thread_environment() { #endif mysql_cond_init(key_COND_manager, &COND_manager); + mysql_cond_init(key_COND_slave_stats_daemon, &COND_slave_stats_daemon); mysql_mutex_init(key_LOCK_server_started, &LOCK_server_started, MY_MUTEX_INIT_FAST); mysql_cond_init(key_COND_server_started, &COND_server_started); @@ -12608,6 +12624,7 @@ static PSI_mutex_info all_server_mutexes[]= { &key_LOCK_handler_count, "LOCK_handler_count", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, #endif { &key_LOCK_manager, "LOCK_manager", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, + { &key_LOCK_slave_stats_daemon, "LOCK_slave_stats_daemon", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_LOCK_prepared_stmt_count, "LOCK_prepared_stmt_count", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_LOCK_replica_list, "LOCK_replica_list", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_LOCK_sql_replica_skip_counter, "LOCK_sql_replica_skip_counter", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, @@ -12771,6 +12788,7 @@ static PSI_cond_info all_server_conds[]= { &key_COND_handler_count, "COND_handler_count", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, #endif { &key_COND_manager, "COND_manager", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, + { &key_COND_slave_stats_daemon, "COND_slave_stats_daemon", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_COND_server_started, "COND_server_started", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, #if !defined(_WIN32) { &key_COND_socket_listener_active, "COND_socket_listener_active", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, @@ -12803,6 +12821,7 @@ static PSI_cond_info all_server_conds[]= PSI_thread_key key_thread_bootstrap; PSI_thread_key key_thread_handle_manager; +PSI_thread_key key_thread_handle_slave_stats_daemon; PSI_thread_key key_thread_one_connection; PSI_thread_key key_thread_compress_gtid_table; PSI_thread_key key_thread_parser_service; @@ -12819,6 +12838,7 @@ static PSI_thread_info all_server_threads[]= #endif /* _WIN32 */ { &key_thread_bootstrap, "bootstrap", "boot", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_thread_handle_manager, "manager", "handle_mgr", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, + { &key_thread_handle_slave_stats_daemon, "slave_stats_daemon", "slave_stat", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_thread_main, "main", "main", PSI_FLAG_SINGLETON, 0, PSI_DOCUMENT_ME}, { &key_thread_one_connection, "one_connection", "connection", PSI_FLAG_USER | PSI_FLAG_NO_SEQNUM, 0, PSI_DOCUMENT_ME}, diff --git a/sql/mysqld.h b/sql/mysqld.h index c8564eea4f51..b73fde622cff 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -512,6 +512,9 @@ inline ulonglong microseconds_to_my_timer(double when) { } extern bool is_slave; +extern std::atomic slave_stats_daemon_thread_counter; +extern uint write_stats_count; +extern ulong write_stats_frequency; extern bool read_only_slave; extern bool flush_only_old_table_cache_entries; extern ulong stored_program_cache_size; @@ -690,6 +693,7 @@ extern PSI_cond_key key_commit_order_manager_cond; extern PSI_cond_key key_COND_group_replication_connection_cond_var; extern PSI_thread_key key_thread_bootstrap; extern PSI_thread_key key_thread_handle_manager; +extern PSI_thread_key key_thread_handle_slave_stats_daemon; extern PSI_thread_key key_thread_one_connection; extern PSI_thread_key key_thread_compress_gtid_table; extern PSI_thread_key key_thread_parser_service; @@ -847,7 +851,7 @@ extern PSI_statement_info sql_statement_info[(uint)SQLCOM_END + 1]; Statement instrumentation keys (com). The last entry, at [COM_END], is for packet errors. */ -extern PSI_statement_info com_statement_info[(uint)COM_END + 1]; +extern PSI_statement_info com_statement_info[(uint)COM_TOP_END]; /** Statement instrumentation key for replication. @@ -897,6 +901,7 @@ extern mysql_mutex_t LOCK_status; extern mysql_mutex_t LOCK_uuid_generator; extern mysql_mutex_t LOCK_crypt; extern mysql_mutex_t LOCK_manager; +extern mysql_mutex_t LOCK_slave_stats_daemon; extern mysql_mutex_t LOCK_global_system_variables; extern mysql_mutex_t LOCK_user_conn; extern mysql_mutex_t LOCK_log_throttle_qni; @@ -926,6 +931,7 @@ extern mysql_mutex_t LOCK_global_conn_mem_limit; extern mysql_cond_t COND_server_started; extern mysql_cond_t COND_compress_gtid_table; extern mysql_cond_t COND_manager; +extern mysql_cond_t COND_slave_stats_daemon; extern mysql_rwlock_t LOCK_sys_init_connect; extern mysql_rwlock_t LOCK_sys_init_replica; diff --git a/sql/protocol_classic.cc b/sql/protocol_classic.cc index 315b2596c692..277bb35b30c8 100644 --- a/sql/protocol_classic.cc +++ b/sql/protocol_classic.cc @@ -3038,7 +3038,7 @@ int Protocol_classic::get_command(COM_DATA *com_data, *cmd = (enum enum_server_command)(uchar)input_raw_packet[0]; - if (*cmd >= COM_END && *cmd != COM_QUERY_ATTRS) + if (*cmd >= COM_END && *cmd <= COM_TOP_BEGIN) *cmd = COM_END; // Wrong command assert(input_packet_length); diff --git a/sql/rpl_replica.cc b/sql/rpl_replica.cc index 74ffa2b229e6..20051f04cec9 100644 --- a/sql/rpl_replica.cc +++ b/sql/rpl_replica.cc @@ -149,6 +149,7 @@ #include "sql/rpl_rli_pdb.h" // Slave_worker #include "sql/rpl_trx_boundary_parser.h" #include "sql/rpl_utility.h" +#include "sql/slave_stats_daemon.h" // stop_handle_slave_stats_daemon, start_handle_slave_stats_daemon #include "sql/sql_backup_lock.h" // is_instance_backup_locked #include "sql/sql_class.h" // THD #include "sql/sql_const.h" @@ -3422,6 +3423,145 @@ static int write_ignored_events_info_to_relay_log(THD *thd, Master_info *mi) { return error; } +/** + Returns slave lag duration relative to master. + @param mi Pointer to Master_info object for the IO thread. + + @retval pair(sec_behind_master, milli_second_behind_master) + This function sets above values to -1 to represent nulls +*/ +std::pair get_time_lag_behind_master(Master_info *mi) { + longlong sec_behind_master = -1; + longlong milli_sec_behind_master = -1; + /* + The pseudo code to compute Seconds_Behind_Master: + if (SQL thread is running) + { + if (SQL thread processed all the available relay log) + { + if (IO thread is running) + print 0; + else + print NULL; + } + else + compute Seconds_Behind_Master; + } + else + print NULL; + */ + + bool sbm_is_null = false; + bool sbm_is_zero = false; + if (mi->rli->slave_running) { + time_t now = time(0); + /* + Check if SQL thread is at the end of relay log + Checking should be done using two conditions + condition1: compare the log positions and + condition2: compare the file names (to handle rotation case) + */ + if (reset_seconds_behind_master && + (mi->get_master_log_pos() == mi->rli->get_group_master_log_pos()) && + (!strcmp(mi->get_master_log_name(), + mi->rli->get_group_master_log_name()))) { + if (mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) + sec_behind_master = 0LL; + else + sec_behind_master = -1; + sbm_is_zero = mi->slave_running == MYSQL_SLAVE_RUN_CONNECT; + sbm_is_null = !sbm_is_zero; + } else { + long time_diff = ((long)(now - mi->rli->last_master_timestamp) - + mi->clock_diff_with_master); + /* + Apparently on some systems time_diff can be <0. Here are possible + reasons related to MySQL: + - the master is itself a slave of another master whose time is ahead. + - somebody used an explicit SET TIMESTAMP on the master. + Possible reason related to granularity-to-second of time functions + (nothing to do with MySQL), which can explain a value of -1: + assume the master's and slave's time are perfectly synchronized, and + that at slave's connection time, when the master's timestamp is read, + it is at the very end of second 1, and (a very short time later) when + the slave's timestamp is read it is at the very beginning of second + 2. Then the recorded value for master is 1 and the recorded value for + slave is 2. At SHOW SLAVE STATUS time, assume that the difference + between timestamp of slave and rli->last_master_timestamp is 0 + (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. + This confuses users, so we don't go below 0: hence the max(). + + last_master_timestamp == 0 (an "impossible" timestamp 1970) is a + special marker to say "consider we have caught up". + */ + if (mi->rli->last_master_timestamp == 0) { + /* + If the I/O thread is encountering problems during initailization, + then display NULL instead of 0. + */ + sbm_is_zero = mi->slave_running == MYSQL_SLAVE_RUN_CONNECT; + sbm_is_null = !sbm_is_zero; + } + if (sbm_is_null) { + sec_behind_master = -1; + } else { + sec_behind_master = + (longlong)(mi->rli->last_master_timestamp ? max(0L, time_diff) : 0); + } + } + } else { + sec_behind_master = -1; + sbm_is_null = true; + } + + // Milli_Seconds_Behind_Master + if (opt_binlog_trx_meta_data) { + if (sbm_is_null) + milli_sec_behind_master = -1; + else if (sbm_is_zero) + milli_sec_behind_master = 0LL; + else { + ulonglong now_millis = + duration_cast(system_clock::now().time_since_epoch()) + .count(); + // adjust for clock mismatch + now_millis -= mi->clock_diff_with_master * 1000; + milli_sec_behind_master = + now_millis - mi->rli->last_master_timestamp_millis; + } + } + return std::make_pair(sec_behind_master, milli_sec_behind_master); +} + +/** + Send milli_second_behind_master statistic to primary using + COM_SEND_REPLICA_STATISTICS +*/ +int send_replica_statistics_to_master(MYSQL *mysql, Master_info *mi) { + uchar buf[1024]; + uchar *pos = buf; + DBUG_ENTER("send_replica_statistics_to_master"); + + int timestamp = my_time(0); + std::pair time_lag_behind_master = + get_time_lag_behind_master(mi); + longlong milli_sec_behind_master = + max((longlong)time_lag_behind_master.second, (longlong)0); + + int4store(pos, server_id); + pos += 4; + int4store(pos, timestamp); + pos += 4; + int4store(pos, milli_sec_behind_master); + pos += 4; + + if (simple_command(mysql, COM_SEND_REPLICA_STATISTICS, buf, + (size_t)(pos - buf), 0)) { + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} + static int register_slave_on_master(MYSQL *mysql, Master_info *mi, bool *suppress_warnings) { uchar buf[1024], *pos = buf; @@ -3601,6 +3741,8 @@ static void show_slave_status_metadata(mem_root_deque *field_list, sizeof(ulong), MYSQL_TYPE_LONG)); field_list->push_back( new Item_empty_string("Network_Namespace", NAME_LEN + 1)); + field_list->push_back( + new Item_empty_string("Slave_Lag_Stats_Thread_Running", 3)); } /** @@ -3754,99 +3896,22 @@ static bool show_slave_status_send_data(THD *thd, Master_info *mi, protocol->store(mi->ssl_cipher, &my_charset_bin); protocol->store(mi->ssl_key, &my_charset_bin); - /* - The pseudo code to compute Seconds_Behind_Master: - if (SQL thread is running) - { - if (SQL thread processed all the available relay log) - { - if (IO thread is running) - print 0; - else - print NULL; - } - else - compute Seconds_Behind_Master; - } - else - print NULL; - */ - - bool sbm_is_null = false; - bool sbm_is_zero = false; - if (mi->rli->slave_running) { - /* - Check if SQL thread is at the end of relay log - Checking should be done using two conditions - condition1: compare the log positions and - condition2: compare the file names (to handle rotation case) - */ - if (reset_seconds_behind_master && - (mi->get_master_log_pos() == mi->rli->get_group_master_log_pos()) && - (!strcmp(mi->get_master_log_name(), - mi->rli->get_group_master_log_name()))) { - if (mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) - protocol->store(0LL); - else - protocol->store_null(); - sbm_is_zero = mi->slave_running == MYSQL_SLAVE_RUN_CONNECT; - sbm_is_null = !sbm_is_zero; - } else { - long time_diff = ((long)(time(nullptr) - mi->rli->last_master_timestamp) - - mi->clock_diff_with_master); - /* - Apparently on some systems time_diff can be <0. Here are possible - reasons related to MySQL: - - the master is itself a slave of another master whose time is ahead. - - somebody used an explicit SET TIMESTAMP on the master. - Possible reason related to granularity-to-second of time functions - (nothing to do with MySQL), which can explain a value of -1: - assume the master's and slave's time are perfectly synchronized, and - that at slave's connection time, when the master's timestamp is read, - it is at the very end of second 1, and (a very short time later) when - the slave's timestamp is read it is at the very beginning of second - 2. Then the recorded value for master is 1 and the recorded value for - slave is 2. At SHOW REPLICA STATUS time, assume that the difference - between timestamp of slave and rli->last_master_timestamp is 0 - (i.e. they are in the same second), then we get 0-(2-1)=-1 as a result. - This confuses users, so we don't go below 0: hence the max(). + std::pair time_lag_behind_master = + get_time_lag_behind_master(mi); - last_master_timestamp == 0 (an "impossible" timestamp 1970) is a - special marker to say "consider we have caught up". - */ - if (mi->rli->last_master_timestamp == 0) { - /* - If the I/O thread is encountering problems during initailization, - then display NULL instead of 0. - */ - sbm_is_zero = mi->slave_running == MYSQL_SLAVE_RUN_CONNECT; - sbm_is_null = !sbm_is_zero; - } - if (sbm_is_null) { - protocol->store_null(); - } else { - protocol->store((longlong)( - mi->rli->last_master_timestamp ? max(0L, time_diff) : 0)); - } - } - } else { + // Seconds_Behind_Master + if (time_lag_behind_master.first == -1) { protocol->store_null(); - sbm_is_null = true; + } else { + protocol->store(time_lag_behind_master.first); } // Milli_Seconds_Behind_Master if (opt_binlog_trx_meta_data) { - if (sbm_is_null) + if (time_lag_behind_master.second == -1) { protocol->store_null(); - else if (sbm_is_zero) - protocol->store(0LL); - else { - ulonglong now_millis = - duration_cast(system_clock::now().time_since_epoch()) - .count(); - // adjust for clock mismatch - now_millis -= mi->clock_diff_with_master * 1000; - protocol->store(now_millis - mi->rli->last_master_timestamp_millis); + } else { + protocol->store(time_lag_behind_master.second); } } @@ -3928,6 +3993,9 @@ static bool show_slave_status_send_data(THD *thd, Master_info *mi, protocol->store(mi->get_public_key ? 1 : 0); protocol->store(mi->network_namespace_str(), &my_charset_bin); + // slave lag stats daemon running status + protocol->store(slave_stats_daemon_thread_counter > 0 ? "Yes" : "No", + &my_charset_bin); rpl_filter->unlock(); mysql_mutex_unlock(&mi->rli->err_lock); @@ -5572,6 +5640,7 @@ extern "C" void *handle_slave_io(void *arg) { uint retry_count; bool suppress_warnings; int ret; + bool slave_stats_daemon_created = false; Global_THD_manager *thd_manager = Global_THD_manager::get_instance(); // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff my_thread_init(); @@ -5737,6 +5806,10 @@ extern "C" void *handle_slave_io(void *arg) { goto connected; } + if (!slave_stats_daemon_created) { + // start sending secondary lag stats to primary + slave_stats_daemon_created = start_handle_slave_stats_daemon(); + } DBUG_PRINT("info", ("Starting reading binary log from master")); while (!io_slave_killed(thd, mi)) { MYSQL_RPL rpl; @@ -5958,6 +6031,11 @@ extern "C" void *handle_slave_io(void *arg) { // error = 0; err: + if (slave_stats_daemon_created) { + // stop sending secondary lag stats to primary + stop_handle_slave_stats_daemon(); + } + /* If source_connection_auto_failover (async connection failover) is enabled, this server is not a Group Replication SECONDARY and @@ -8681,27 +8759,10 @@ static int safe_connect(THD *thd, MYSQL *mysql, Master_info *mi, port); } -int connect_to_master(THD *thd, MYSQL *mysql, Master_info *mi, bool reconnect, - bool suppress_warnings, const std::string &host, - const uint port, bool is_io_thread) { - int last_errno = -2; // impossible error - ulong err_count = 0; - char llbuff[22]; - char password[MAX_PASSWORD_LENGTH + 1]; - size_t password_size = sizeof(password); - DBUG_TRACE; - set_replica_max_allowed_packet(thd, mysql); -#ifndef NDEBUG - mi->events_until_exit = disconnect_slave_event_count; -#endif - ulong client_flag = CLIENT_REMEMBER_OPTIONS; - - /* Always reset public key to remove cached copy */ - mysql_reset_server_public_key(); - - mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, (char *)&replica_net_timeout); - mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, (char *)&replica_net_timeout); - +/* + method to configure some common mysql options for connection to master +*/ +void configure_master_connection_options(MYSQL *mysql, Master_info *mi) { if (mi->bind_addr[0]) { DBUG_PRINT("info", ("bind_addr: %s", mi->bind_addr)); mysql_options(mysql, MYSQL_OPT_BIND, mi->bind_addr); @@ -8798,6 +8859,30 @@ int connect_to_master(THD *thd, MYSQL *mysql, Master_info *mi, bool reconnect, /* Get public key from master */ DBUG_PRINT("info", ("Set preference to get public key from master")); mysql_options(mysql, MYSQL_OPT_GET_SERVER_PUBLIC_KEY, &mi->get_public_key); +} + +int connect_to_master(THD *thd, MYSQL *mysql, Master_info *mi, bool reconnect, + bool suppress_warnings, const std::string &host, + const uint port, bool is_io_thread) { + int last_errno = -2; // impossible error + ulong err_count = 0; + char llbuff[22]; + char password[MAX_PASSWORD_LENGTH + 1]; + size_t password_size = sizeof(password); + DBUG_TRACE; + set_replica_max_allowed_packet(thd, mysql); +#ifndef NDEBUG + mi->events_until_exit = disconnect_slave_event_count; +#endif + ulong client_flag = CLIENT_REMEMBER_OPTIONS; + + /* Always reset public key to remove cached copy */ + mysql_reset_server_public_key(); + + mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, (char *)&replica_net_timeout); + mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, (char *)&replica_net_timeout); + + configure_master_connection_options(mysql, mi); if (is_io_thread && !mi->is_start_user_configured()) LogErr(WARNING_LEVEL, ER_RPL_SLAVE_INSECURE_CHANGE_MASTER); diff --git a/sql/rpl_replica.h b/sql/rpl_replica.h index a9b86d7a9a0d..bcad789bc678 100644 --- a/sql/rpl_replica.h +++ b/sql/rpl_replica.h @@ -24,6 +24,7 @@ #define RPL_REPLICA_H #include +#include #include #include #include @@ -490,6 +491,8 @@ int flush_master_info(Master_info *mi, bool force, bool need_lock = true, bool skip_repo_persistence = false); void add_replica_skip_errors(const char *arg); void set_replica_skip_errors(char **replica_skip_errors_ptr); +void configure_master_connection_options(MYSQL *mysql, Master_info *mi); +int send_replica_statistics_to_master(MYSQL *mysql, Master_info *mi); int add_new_channel(Master_info **mi, const char *channel); /** Terminates the slave threads according to the given mask. diff --git a/sql/rpl_source.cc b/sql/rpl_source.cc index f468070f0256..d19f25f1e0dd 100644 --- a/sql/rpl_source.cc +++ b/sql/rpl_source.cc @@ -29,6 +29,7 @@ #include #include #include +#include #include "binlog.h" #include "m_ctype.h" @@ -127,6 +128,7 @@ int register_replica(THD *thd, uchar *packet, size_t packet_length) { unique_ptr_my_free si((REPLICA_INFO *)my_malloc( key_memory_REPLICA_INFO, sizeof(REPLICA_INFO), MYF(MY_WME))); if (si == nullptr) return 1; + new (si.get()) REPLICA_INFO; /* 4 bytes for the server id */ if (p + 4 > p_end) { @@ -169,6 +171,73 @@ int register_replica(THD *thd, uchar *packet, size_t packet_length) { return 1; } +SLAVE_STATS::SLAVE_STATS(uchar *packet) { + /* 4 bytes for the server id */ + /* 4 bytes timestamp */ + /* 4 bytes for milli_second_behind_master */ + server_id = uint4korr(packet); + packet += 4; + timestamp = uint4korr(packet); + packet += 4; + milli_sec_behind_master = uint4korr(packet); +} + +/** + Populates slave statistics data-point into the slave_lists hash table. + These stats are sent by slaves to master at regular intervals. + + @return + 0 ok + @return + 1 Error. Error message sent to client +*/ +int store_replica_stats(THD *thd, uchar *packet, uint packet_length) { + if (check_access(thd, REPL_SLAVE_ACL, any_db, nullptr, nullptr, 0, 0)) + return 1; + if (sizeof(SLAVE_STATS) > packet_length) { + my_error(ER_MALFORMED_PACKET, MYF(0)); + return 1; + } + + SLAVE_STATS stats(packet); + + mysql_mutex_lock(&LOCK_replica_list); + auto it = slave_list.find(stats.server_id); + if (it != slave_list.end()) { + REPLICA_INFO *si = it->second.get(); + + // We are over the configured size. Erase older entries first. + while (!si->slave_stats.empty() && + si->slave_stats.size() >= write_stats_count) { + si->slave_stats.pop_back(); + } + + if (write_stats_count > 0) { + si->slave_stats.push_front(stats); + } + } + mysql_mutex_unlock(&LOCK_replica_list); + return 0; +} + +/** + Returns a vector of replica_statistics_row objects to be used to populate + performance_schema.replica_statistics table. +*/ +std::vector get_all_replica_statistics() { + std::vector replica_statistics; + mysql_mutex_lock(&LOCK_replica_list); + for (const auto &key_and_value : slave_list) { + REPLICA_INFO *si = key_and_value.second.get(); + for (const SLAVE_STATS &stats : si->slave_stats) { + replica_statistics.emplace_back(si->server_id, stats.timestamp, + stats.milli_sec_behind_master); + } + } + mysql_mutex_unlock(&LOCK_replica_list); + return replica_statistics; +} + void unregister_replica(THD *thd, bool only_mine, bool need_lock_slave_list) { if (thd->server_id) { if (need_lock_slave_list) diff --git a/sql/rpl_source.h b/sql/rpl_source.h index 3a7a4d5d3c37..c95f57764935 100644 --- a/sql/rpl_source.h +++ b/sql/rpl_source.h @@ -24,7 +24,9 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include +#include #include // std::string +#include #include "libbinlogevents/include/uuid.h" // UUID #include "map_helpers.h" @@ -33,6 +35,7 @@ #include "my_thread_local.h" // my_thread_id #include "mysql_com.h" // USERNAME_LENGTH #include "sql/sql_const.h" // MAX_PASSWORD_LENGTH +#include "storage/perfschema/table_replica_statistics.h" struct Gtid; struct snapshot_info_st; @@ -46,6 +49,18 @@ extern int max_binlog_dump_events; extern bool opt_sporadic_binlog_dump_fail; extern bool opt_show_replica_auth_info; +/* + * SlaveStats struct contains the statistics continuously sent by slaves to the + * master + */ +struct SLAVE_STATS { + int server_id; + int timestamp; + int milli_sec_behind_master; + + explicit SLAVE_STATS(uchar *packet); +}; + struct REPLICA_INFO { uint32 server_id; uint32 rpl_recovery_rank, master_id; @@ -53,6 +68,7 @@ struct REPLICA_INFO { char user[USERNAME_LENGTH + 1]; char password[MAX_PASSWORD_LENGTH + 1]; uint16 port; + std::list slave_stats; THD *thd; binary_log::Uuid replica_uuid; bool valid_replica_uuid; @@ -62,6 +78,7 @@ using thd_to_slave_info_container = malloc_unordered_map; thd_to_slave_info_container copy_slaves(); bool is_semi_sync_slave(THD *thd, bool need_lock = true); +int store_replica_stats(THD *thd, uchar *packet, uint packet_length); int register_replica(THD *thd, uchar *packet, size_t packet_length); void unregister_replica(THD *thd, bool only_mine, bool need_lock_slave_list); bool show_replicas(THD *thd); @@ -134,6 +151,8 @@ void mysql_binlog_send(THD *thd, char *log_ident, my_off_t pos, bool reset_master(THD *thd, bool unlock_read_lock); +std::vector get_all_replica_statistics(); + class user_var_entry; /** Read a user variable that may exist under two different names. diff --git a/sql/slave_stats_daemon.cc b/sql/slave_stats_daemon.cc new file mode 100644 index 000000000000..3545f93c9283 --- /dev/null +++ b/sql/slave_stats_daemon.cc @@ -0,0 +1,249 @@ +/* Copyright (c) 2010, 2019, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include +#include +#include +#include +#include + +#include "sql/log.h" +#include "sql/protocol_classic.h" +#include "sql/rpl_mi.h" +#include "sql/rpl_msr.h" // Multisource_info +#include "sql/rpl_replica.h" +#include "sql/slave_stats_daemon.h" +#include "sql/sql_base.h" +#include "sql/sql_show.h" +#include "sql/srv_session.h" + +#if OPENSSL_VERSION_NUMBER >= 0x10100000L +// Function removed after OpenSSL 1.1.0 +#define ERR_remove_state(x) +#endif + +/* + * The Slave stats daemon thread is responsible for + * continuously sending lag statistics from slaves to masters + */ + +my_thread_t slave_stats_daemon_thread; +mysql_mutex_t LOCK_slave_stats_daemon; +mysql_cond_t COND_slave_stats_daemon; + +/* connection/read timeout in seconds*/ +const int REPLICA_STATS_NET_TIMEOUT = 5; + +static bool abort_slave_stats_daemon = false; + +static bool connected_to_master = false; + +/** + Create and initialize the mysql object, and connect to the + master. + + @retval true if connection successful + @retval false otherwise. +*/ +static bool safe_connect_slave_stats_thread_to_master(MYSQL *&mysql, + Master_info *active_mi) { + if (mysql != nullptr) { + mysql_close(mysql); + } + mysql = mysql_init(nullptr); + if (!mysql) { + return false; + } + mysql_options(mysql, MYSQL_OPT_CONNECT_TIMEOUT, + (const char *)&REPLICA_STATS_NET_TIMEOUT); + mysql_options(mysql, MYSQL_OPT_READ_TIMEOUT, + (const char *)&REPLICA_STATS_NET_TIMEOUT); + configure_master_connection_options(mysql, active_mi); + + char pass[MAX_PASSWORD_LENGTH + 1]; + size_t password_size = sizeof(pass); + if (active_mi->get_password(pass, &password_size)) { + return false; + } + + if (!mysql_real_connect(mysql, active_mi->host, active_mi->get_user(), pass, + 0, active_mi->port, 0, 0)) { + return false; + } + return true; +} + +extern "C" { +static void *handle_slave_stats_daemon(void *arg MY_ATTRIBUTE((unused))) { + THD *thd = nullptr; + int error = 0; + struct timespec abstime; + + DBUG_ENTER("handle_slave_stats_daemon"); + + slave_stats_daemon_thread = my_thread_self(); + + MYSQL *mysql = nullptr; + Master_info *active_mi; + while (true) { + mysql_mutex_lock(&LOCK_slave_stats_daemon); + set_timespec(&abstime, write_stats_frequency); + while ((!error || error == EINTR) && !abort_slave_stats_daemon) { + /* + write_stats_frequency is set to 0. Do not send stats to master. + Wait until a signal is received either for aborting the thread or for + updating write_stats_frequency. + */ + if (write_stats_frequency == 0) { + error = + mysql_cond_wait(&COND_slave_stats_daemon, &LOCK_slave_stats_daemon); + } else { + /* + wait for write_stats_frequency seconds before sending next set + of slave lag statistics + */ + error = mysql_cond_timedwait(&COND_slave_stats_daemon, + &LOCK_slave_stats_daemon, &abstime); + } + } + + mysql_mutex_unlock(&LOCK_slave_stats_daemon); + + if (abort_slave_stats_daemon) break; + + if (error == ETIMEDOUT) { + // Initialize connection thd, if not already done. + if (thd == nullptr) { + my_thread_init(); + thd = new THD; + thd->set_new_thread_id(); + THD_CHECK_SENTRY(thd); + thd->thread_stack = (char *)&thd; + my_net_init(thd->get_protocol_classic()->get_net(), 0); + thd->store_globals(); + } + + channel_map.rdlock(); + active_mi = channel_map.get_default_channel_mi(); + channel_map.unlock(); + // If not connected to current master, try connection. If not + // successful, try again in next cycle + + if (!connected_to_master) { + connected_to_master = + safe_connect_slave_stats_thread_to_master(mysql, active_mi); + if (connected_to_master) { + DBUG_PRINT("info", + ("Slave Stats Daemon: connected to master '%s@%s:%d'", + active_mi->get_user(), active_mi->host, active_mi->port)); + } else { + DBUG_PRINT( + "info", + ("Slave Stats Daemon: Couldn't connect to master '%s@%s:%d', " + "will try again during next cycle, (Error: %s)", + active_mi->get_user(), active_mi->host, active_mi->port, + mysql_error(mysql))); + } + } + if (connected_to_master && + active_mi->slave_running == MYSQL_SLAVE_RUN_CONNECT) { + if (send_replica_statistics_to_master(mysql, active_mi)) { + DBUG_PRINT("info", ("Slave Stats Daemon: Failed to send lag " + "statistics, resetting connection, (Error: %s)", + mysql_error(mysql))); + connected_to_master = false; + } + } + error = 0; + } + } + mysql_close(mysql); + mysql = nullptr; + connected_to_master = false; + if (thd != nullptr) { + net_end(thd->get_protocol_classic()->get_net()); + thd->release_resources(); + delete (thd); + } + ERR_remove_state(0); + assert(slave_stats_daemon_thread_counter > 0); + slave_stats_daemon_thread_counter--; + my_thread_end(); + return (nullptr); +} +} // extern "C" + +/* Start handle Slave Stats Daemon thread */ +bool start_handle_slave_stats_daemon() { + DBUG_ENTER("start_handle_slave_stats_daemon"); + + channel_map.rdlock(); + if (channel_map.get_num_instances() != 1) { + // more than one channels exists for this slave. We only support + // single source slave topologies for now. Skip creating the thread. + sql_print_information( + "Number of channels = %lu. There should be only one channel" + " with slave_stats_daemon. Not creating the thread.", + channel_map.get_num_instances()); + channel_map.unlock(); + DBUG_RETURN(false); + } + channel_map.unlock(); + + my_thread_handle thread_handle; + slave_stats_daemon_thread_counter++; + int error = + mysql_thread_create(key_thread_handle_slave_stats_daemon, &thread_handle, + &connection_attrib, handle_slave_stats_daemon, 0); + if (error) { + sql_print_warning("Can't create Slave Stats Daemon thread (errno= %d)", + error); + assert(slave_stats_daemon_thread_counter > 0); + slave_stats_daemon_thread_counter--; + DBUG_RETURN(false); + } + sql_print_information("Successfully created Slave Stats Daemon thread: 0x%lx", + (ulong)slave_stats_daemon_thread); + DBUG_RETURN(true); +} + +/* Initiate shutdown of handle Slave Stats Daemon thread */ +void stop_handle_slave_stats_daemon() { + DBUG_ENTER("stop_handle_slave_stats_daemon"); + abort_slave_stats_daemon = true; + mysql_mutex_lock(&LOCK_slave_stats_daemon); + sql_print_information("Shutting down Slave Stats Daemon thread: 0x%lx", + (ulong)slave_stats_daemon_thread); + // there must be at most one slave_stats_daemon thread to stop + assert(slave_stats_daemon_thread_counter <= 1); + mysql_cond_broadcast(&COND_slave_stats_daemon); + mysql_mutex_unlock(&LOCK_slave_stats_daemon); + while (slave_stats_daemon_thread_counter > 0) { + // wait for the thread to finish, sleep for 10ms + my_sleep(10000); + } + // Reset abort_slave_stats_daemon so slave_stats_daemon can be spawned in + // future + abort_slave_stats_daemon = false; + DBUG_VOID_RETURN; +} diff --git a/sql/slave_stats_daemon.h b/sql/slave_stats_daemon.h new file mode 100644 index 000000000000..5b17c81ffd13 --- /dev/null +++ b/sql/slave_stats_daemon.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2010, 2019, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef SLAVE_STATS_DAEMON_H +#define SLAVE_STATS_DAEMON_H + +extern bool start_handle_slave_stats_daemon(); +extern void stop_handle_slave_stats_daemon(); + +#endif diff --git a/sql/sql_audit.cc b/sql/sql_audit.cc index e96acccb693c..c561b42d419e 100644 --- a/sql/sql_audit.cc +++ b/sql/sql_audit.cc @@ -847,7 +847,8 @@ class Ignore_command_start_error_handler : public Audit_error_handler { if (command == COM_QUIT || command == COM_PING || command == COM_SLEEP || /* Deprecated commands from here. */ command == COM_CONNECT || command == COM_TIME || - command == COM_DELAYED_INSERT || command == COM_END) { + command == COM_DELAYED_INSERT || command == COM_END || + command == COM_TOP_BEGIN || command == COM_TOP_END) { return true; } diff --git a/sql/sql_class.h b/sql/sql_class.h index a214aa597d56..60944bd33fd7 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -106,7 +106,8 @@ #include "sql/sql_plugin_ref.h" #include "sql/sys_vars_resource_mgr.h" // Session_sysvar_resource_manager #include "sql/system_variables.h" // system_variables -#include "sql/transaction_info.h" // Ha_trx_info +#include "sql/table.h" +#include "sql/transaction_info.h" // Ha_trx_info #include "sql/xa.h" #include "sql_db.h" #include "sql_string.h" diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 81f73e73f350..6a001dadf051 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -223,7 +223,7 @@ static void sql_kill(THD *thd, my_thread_id id, bool only_kill_query); const char *hlc_ts_lower_bound = "hlc_ts_lower_bound"; -const std::string Command_names::m_names[] = { +const std::string Command_names::m_names[COM_TOP_END] = { "Sleep", "Quit", "Init DB", @@ -258,7 +258,228 @@ const std::string Command_names::m_names[] = { "Reset Connection", "clone", "Group Replication Data Stream subscription", - "Error" // Last command number + "Error", // Last command number + "", + "", + "", + "", + "", + "", // 40 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 50 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 60 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 70 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 80 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 90 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 100 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 110 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 120 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 130 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 140 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 150 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 160 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 170 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 180 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 190 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 200 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 210 + "", + "", + "", + "", + "", + "", + "", + "", + "", // 220 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 230 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 240 + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", // 250 + "", + "", + "Error", // COM_TOP_BEGIN + "Send Replica Statistics", // COM_SEND_REPLICA_STATISTICS + "Query Attrs" // COM_QUERY_ATTRS }; const std::string &Command_names::translate(const System_variables &sysvars) { @@ -433,7 +654,7 @@ bool stmt_causes_implicit_commit(const THD *thd, uint mask) { */ uint sql_command_flags[SQLCOM_END + 1]; -uint server_command_flags[COM_END + 1]; +uint server_command_flags[COM_TOP_END]; void init_sql_command_flags(void) { /* Initialize the server command flags array. */ @@ -1867,6 +2088,13 @@ bool dispatch_command(THD *thd, const COM_DATA *com_data, my_ok(thd); break; } + case COM_SEND_REPLICA_STATISTICS: { + if (!store_replica_stats( + thd, thd->get_protocol_classic()->get_raw_packet(), + thd->get_protocol_classic()->get_packet_length())) + my_ok(thd); + break; + } case COM_RESET_CONNECTION: { thd->status_var.com_other++; thd->cleanup_connection(); diff --git a/sql/sql_parse.h b/sql/sql_parse.h index 2b54a7beeed9..93f656fef5ee 100644 --- a/sql/sql_parse.h +++ b/sql/sql_parse.h @@ -160,7 +160,7 @@ class Command_names { Array indexed by enum_server_command, where each element is a description string. */ - static const std::string m_names[]; + static const std::string m_names[COM_TOP_END]; /** Command whose name depends on @@terminology_use_previous. @@ -201,7 +201,7 @@ class Command_names { */ static enum_server_command int_to_cmd(int cmd) { assert(cmd >= 0); - assert(cmd <= COM_END); + assert(cmd < COM_TOP_END); return static_cast(cmd); } diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 6ee59bebd9ae..0b8f1eae6373 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -797,6 +797,7 @@ static Sys_var_long Sys_pfs_events_stages_history_size( Variable performance_schema_max_statement_classes. The default number of statement classes is the sum of: - COM_END for all regular "statement/com/...", + - COM_MAX - COM_TOP_END - 1 for commands residing at the top end, - 1 for "statement/com/new_packet", for unknown enum_server_command - 1 for "statement/com/Error", for invalid enum_server_command - SQLCOM_END for all regular "statement/sql/...", @@ -811,7 +812,8 @@ static Sys_var_ulong Sys_pfs_max_statement_classes( "Maximum number of statement instruments.", READ_ONLY GLOBAL_VAR(pfs_param.m_statement_class_sizing), CMD_LINE(REQUIRED_ARG), VALID_RANGE(0, 256), - DEFAULT((ulong)SQLCOM_END + (ulong)COM_END + 5 + + DEFAULT((ulong)SQLCOM_END + (ulong)COM_END + + (COM_TOP_END - COM_TOP_BEGIN - 1) + 5 + SP_PSI_STATEMENT_INFO_COUNT + CLONE_PSI_STATEMENT_COUNT), BLOCK_SIZE(1), PFS_TRAILING_PROPERTIES); @@ -8771,3 +8773,31 @@ static Sys_var_ulonglong Sys_apply_log_retention_duration( "Minimum duration (mins) that apply logs need to be retained.", GLOBAL_VAR(apply_log_retention_duration), CMD_LINE(OPT_ARG), VALID_RANGE(0, ULONG_LONG_MAX), DEFAULT(15), BLOCK_SIZE(1)); + +static Sys_var_uint Sys_write_stats_count( + "write_stats_count", + "Maximum number of most recent data points to be collected for " + "information_schema.write_statistics & " + "information_schema.replica_statistics " + "time series.", + GLOBAL_VAR(write_stats_count), CMD_LINE(OPT_ARG), VALID_RANGE(0, UINT_MAX), + DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(nullptr), + ON_UPDATE(nullptr)); + +/* Update the time interval for collecting write statistics. Signal + * the thread to send replica lag stats if it is blocked on interval update */ +static bool update_write_stats_frequency(sys_var *, THD *, enum_var_type) { + mysql_mutex_lock(&LOCK_slave_stats_daemon); + mysql_cond_signal(&COND_slave_stats_daemon); + mysql_mutex_unlock(&LOCK_slave_stats_daemon); + + return false; // success +} + +static Sys_var_ulong Sys_write_stats_frequency( + "write_stats_frequency", + "This variable determines the frequency(seconds) at which write " + "stats and replica lag stats are collected on primaries", + GLOBAL_VAR(write_stats_frequency), CMD_LINE(OPT_ARG), + VALID_RANGE(0, LONG_TIMEOUT), DEFAULT(0), BLOCK_SIZE(1), NO_MUTEX_GUARD, + NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(update_write_stats_frequency)); diff --git a/sql/table.cc b/sql/table.cc index 4839e0df2239..07395cd3d2c2 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -6175,6 +6175,15 @@ void TABLE::set_keyread(bool flag) { } } +void TABLE::set_storage_handler(handler *file_arg) { + // Ensure consistent call order + assert((file == nullptr && file_arg != nullptr) || + (file != nullptr && file_arg == nullptr)); + assert(!is_created()); + assert(file_arg->inited == handler::NONE); + file = file_arg; +} + void TABLE::set_created() { if (created) return; if (key_read) file->ha_extra(HA_EXTRA_KEYREAD); diff --git a/sql/table.h b/sql/table.h index 5722b3ba2710..126efa8531fc 100644 --- a/sql/table.h +++ b/sql/table.h @@ -52,7 +52,6 @@ #include "sql/mdl.h" // MDL_wait_for_subgraph #include "sql/mem_root_array.h" #include "sql/opt_costmodel.h" // Cost_model_table -#include "sql/partition_info.h" #include "sql/record_buffer.h" // Record_buffer #include "sql/sql_bitmap.h" // Bitmap #include "sql/sql_const.h" @@ -1893,14 +1892,8 @@ struct TABLE { bool has_storage_handler() const { return file != nullptr; } /// Set storage handler for temporary table - void set_storage_handler(handler *file_arg) { - // Ensure consistent call order - assert((file == nullptr && file_arg != nullptr) || - (file != nullptr && file_arg == nullptr)); - assert(!is_created()); - assert(file_arg->inited == handler::NONE); - file = file_arg; - } + void set_storage_handler(handler *file_arg); + /// Return true if table is instantiated, and false otherwise. bool is_created() const { return created; } diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt index ea83c3269679..6d3709b85580 100644 --- a/storage/perfschema/CMakeLists.txt +++ b/storage/perfschema/CMakeLists.txt @@ -183,6 +183,7 @@ table_session_connect.h table_session_connect_attrs.h table_session_account_connect_attrs.h table_session_query_attrs.h +table_replica_statistics.h table_replication_asynchronous_connection_failover.h table_replication_connection_configuration.h table_replication_group_members.h @@ -351,6 +352,7 @@ table_session_connect.cc table_session_connect_attrs.cc table_session_account_connect_attrs.cc table_session_query_attrs.cc +table_replica_statistics.cc table_replication_asynchronous_connection_failover.cc table_replication_connection_configuration.cc table_replication_group_members.cc diff --git a/storage/perfschema/ha_perfschema.cc b/storage/perfschema/ha_perfschema.cc index 7df12195e95a..c2054c0686bf 100644 --- a/storage/perfschema/ha_perfschema.cc +++ b/storage/perfschema/ha_perfschema.cc @@ -90,7 +90,7 @@ */ static_assert((PFS_DD_VERSION <= MYSQL_VERSION_ID) || - ((PFS_DD_VERSION == 80028007) && (MYSQL_VERSION_ID == 80028)), + ((PFS_DD_VERSION == 80028008) && (MYSQL_VERSION_ID == 80028)), "This release can not use a version number from the future"); class KEY; diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index f2a3abafc062..c61d428b067e 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -3296,7 +3296,7 @@ void pfs_set_thread_command_vc(int command) { PFS_thread *pfs = my_thread_get_THR_PFS(); assert(command >= 0); - assert(command <= (int)COM_END); + assert(command <= (int)COM_END || command > (int)COM_TOP_BEGIN); if (likely(pfs != nullptr)) { pfs->m_command = command; diff --git a/storage/perfschema/pfs_dd_version.h b/storage/perfschema/pfs_dd_version.h index 28b42e3a75bd..f9ad76ba1ebb 100644 --- a/storage/perfschema/pfs_dd_version.h +++ b/storage/perfschema/pfs_dd_version.h @@ -244,10 +244,13 @@ - add compilation cpu time to statement statistics - add elapsed time to statement statistics + 80028-008: + - add replica_statistics table + The last three digits reprents Facebook specific MySQL Schema changes. - Version published is now 80028-007. i.e. 8.0.28 Facebook schema change no. 7. + Version published is now 80028-008. i.e. 8.0.28 Facebook schema change no. 8. */ -static const uint PFS_DD_VERSION = 80028007; +static const uint PFS_DD_VERSION = 80028008; #endif /* PFS_DD_VERSION_H */ diff --git a/storage/perfschema/pfs_engine_table.cc b/storage/perfschema/pfs_engine_table.cc index 7823d0a4dad7..18af80bcfa8b 100644 --- a/storage/perfschema/pfs_engine_table.cc +++ b/storage/perfschema/pfs_engine_table.cc @@ -121,6 +121,7 @@ #include "storage/perfschema/table_replication_applier_status_by_worker.h" /* For replication related perfschema tables. */ #include "storage/perfschema/table_log_status.h" +#include "storage/perfschema/table_replica_statistics.h" #include "storage/perfschema/table_replication_asynchronous_connection_failover.h" #include "storage/perfschema/table_replication_connection_configuration.h" #include "storage/perfschema/table_replication_connection_status.h" @@ -562,6 +563,7 @@ static PFS_engine_table_share *all_shares[] = { &table_session_connect_attrs::m_share, &table_session_account_connect_attrs::m_share, &table_session_query_attrs::m_share, + &table_replica_statistics::m_share, &table_keyring_keys::s_share, diff --git a/storage/perfschema/table_replica_statistics.cc b/storage/perfschema/table_replica_statistics.cc new file mode 100644 index 000000000000..f94965e17920 --- /dev/null +++ b/storage/perfschema/table_replica_statistics.cc @@ -0,0 +1,149 @@ +/* Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + @file storage/perfschema/table_replica_statistics.h + Performance schema replica_statistics table. +*/ + +#include "storage/perfschema/table_replica_statistics.h" + +#include "sql/field.h" +#include "sql/mysqld_thd_manager.h" +#include "sql/plugin_table.h" +#include "sql/rpl_source.h" +#include "sql/sql_class.h" +#include "sql/table.h" +#include "storage/perfschema/pfs_buffer_container.h" + +#include "storage/perfschema/table_helper.h" + +THR_LOCK table_replica_statistics::m_table_lock; +std::atomic table_replica_statistics::m_most_recent_size(0); + +Plugin_table table_replica_statistics::m_table_def( + /* Schema name */ + "performance_schema", + /* Name */ + "replica_statistics", + /* Definition */ + " SERVER_ID BIGINT unsigned NOT NULL,\n" + " TIMESTAMP TIMESTAMP NOT NULL default 0,\n" + " MILLI_SEC_BEHIND_MASTER BIGINT unsigned NOT NULL\n", + /* Options */ + " ENGINE=PERFORMANCE_SCHEMA", + /* Tablespace */ + nullptr); + +PFS_engine_table_share table_replica_statistics::m_share = { + &pfs_readonly_acl, + table_replica_statistics::create, + NULL, /* write_row */ + NULL, /* delete_all_rows */ + table_replica_statistics::get_row_count, + sizeof(PFS_simple_index), + &table_replica_statistics::m_table_lock, + &table_replica_statistics::m_table_def, + false, /* perpetual */ + PFS_engine_table_proxy(), + {0}, + false /* m_in_purgatory */ +}; + +enum replica_statistics_field_offset { + FO_SERVER_ID, + FO_TIMESTAMP, + FO_MILLI_SEC_BEHIND_MASTER, +}; + +table_replica_statistics::table_replica_statistics() + : PFS_engine_table(&m_share, &m_pos), m_pos(0) { + m_all_rows = get_all_replica_statistics(); + table_replica_statistics::m_most_recent_size = m_all_rows.size(); + m_current_row = nullptr; +} + +PFS_engine_table *table_replica_statistics::create(PFS_engine_table_share *) { + return new table_replica_statistics(); +} + +ha_rows table_replica_statistics::get_row_count(void) { + /* + To hint the optimizer we return the most recent size of + the table when we last loaded the stats + */ + return m_most_recent_size; +} + +void table_replica_statistics::reset_position(void) { m_pos.set_at(0u); } + +int table_replica_statistics::rnd_next(void) { + if (m_pos.m_index >= m_all_rows.size()) { + m_current_row = nullptr; + return HA_ERR_END_OF_FILE; + } + m_current_row = &m_all_rows[m_pos.m_index]; + m_pos.next(); + return 0; +} + +int table_replica_statistics::rnd_pos(const void *pos) { + set_position(pos); + if (m_pos.m_index >= m_all_rows.size()) { + m_current_row = nullptr; + return HA_ERR_RECORD_DELETED; + } + m_current_row = &m_all_rows[m_pos.m_index]; + return 0; +} + +int table_replica_statistics::read_row_values(TABLE *table, unsigned char *buf, + Field **fields, bool read_all) { + Field *f; + const auto &curr_row = *m_current_row; + + /* Set the null bits */ + assert(table->s->null_bytes == 1); + buf[0] = 0; + + for (; (f = *fields); fields++) { + if (read_all || bitmap_is_set(table->read_set, f->field_index())) { + switch (f->field_index()) { + case FO_SERVER_ID: + set_field_ulonglong(f, curr_row.server_id()); + break; + case FO_TIMESTAMP: { + // convert to micro time + set_field_timestamp(f, curr_row.timestamp() * 1000000); + } break; + case FO_MILLI_SEC_BEHIND_MASTER: { + set_field_ulonglong(f, curr_row.milli_sec_behind_master()); + } break; + default: + assert(false); + } + } + } + + return 0; +} diff --git a/storage/perfschema/table_replica_statistics.h b/storage/perfschema/table_replica_statistics.h new file mode 100644 index 000000000000..ab6e61054b05 --- /dev/null +++ b/storage/perfschema/table_replica_statistics.h @@ -0,0 +1,101 @@ +/* Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is also distributed with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have included with MySQL. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + @file storage/perfschema/table_replica_statistics.h + Performance schema replica_statistics table. +*/ + +#ifndef TABLE_REPLICA_STATISTICS_H +#define TABLE_REPLICA_STATISTICS_H + +#include + +#include "storage/perfschema/pfs_engine_table.h" + +class replica_statistics_row { + public: + replica_statistics_row(const ulonglong server_id, const ulonglong timestamp, + const ulonglong milli_sec_behind_master) + : m_server_id(server_id), + m_timestamp(timestamp), + m_milli_sec_behind_master(milli_sec_behind_master) {} + + /* Disabled copy. */ + replica_statistics_row(replica_statistics_row &) = delete; + replica_statistics_row &operator=(replica_statistics_row &) = delete; + + /* Allow std::move copies. */ + replica_statistics_row(replica_statistics_row &&) = default; + replica_statistics_row &operator=(replica_statistics_row &&) = default; + + public: + ulonglong server_id() const { return m_server_id; } + ulonglong timestamp() const { return m_timestamp; } + ulonglong milli_sec_behind_master() const { + return m_milli_sec_behind_master; + } + + private: + ulonglong m_server_id; + ulonglong m_timestamp; + ulonglong m_milli_sec_behind_master; +}; + +class table_replica_statistics : public PFS_engine_table { + public: + /** Table share */ + static PFS_engine_table_share m_share; + static PFS_engine_table *create(PFS_engine_table_share *); + static ha_rows get_row_count(); + + void reset_position(void) override; + int rnd_next() override; + int rnd_pos(const void *pos) override; + + /** Captures the most recent size of table in static context. + * To be used as the estimated return value for get_row_count **/ + static std::atomic m_most_recent_size; + + protected: + int read_row_values(TABLE *table, unsigned char *buf, Field **fields, + bool read_all) override; + + private: + table_replica_statistics(); + + private: + /** Current position. */ + PFS_simple_index m_pos; + + std::vector m_all_rows; + const replica_statistics_row *m_current_row; + + /** Table share lock. */ + static THR_LOCK m_table_lock; + + /** Table definition. */ + static Plugin_table m_table_def; +}; + +#endif diff --git a/storage/perfschema/table_threads.cc b/storage/perfschema/table_threads.cc index ecaad1cc10f2..e8e2a087f861 100644 --- a/storage/perfschema/table_threads.cc +++ b/storage/perfschema/table_threads.cc @@ -307,7 +307,8 @@ int table_threads::make_row(PFS_thread *pfs) { /* Dirty read, sanitize the command. */ m_row.m_command = pfs->m_command; - if ((m_row.m_command < 0) || (m_row.m_command > COM_END)) { + if ((m_row.m_command < 0) || + (m_row.m_command > COM_END && m_row.m_command < COM_TOP_BEGIN)) { m_row.m_command = COM_END; }