fix flaky raft testcase

Summary: There are multiple issues for MTR: 1. in sql/rpl_binlog_sender.cc, if secondaries IO thread receives fatal_error, it will quit IO thread instead of reconnect. use unknown error so that secondary IO thread can try to reconnect 2. mtr.add_suppression() call: mtr.add_suppression() will execute an insert mysql statement into mtr.test_suppressions table and mtr.test_suppressions table doesn't contain primary key, thus during idempotent recovery, secondary will fail to execute mtr.add_suppression() due to missing PK. Try to move all mtr.add_suppression() at the end of testcase to workaround idempotent recovery failure. 3. When promotion, use raft_promote_to_leader.inc instead of `set rpl_raft_new_leader_uuid`, since raft_promote_to_leader will wait the new primary state becomes writeable 4. pass specific warning instead of '.*' to mtr.add_supression 5. etc Reviewed By: Pushapgl, bhatvinay Differential Revision: D28774820 fbshipit-source-id: c898035f7ac
percona · Jun 1, 2021 · 16b2705 · 16b2705
1 parent 5b302cb
commit 16b2705
Show file tree

Hide file tree

Showing 16 changed files with 194 additions and 109 deletions.
diff --git a/mysql-test/include/rpl_end_raft.inc b/mysql-test/include/rpl_end_raft.inc
@@ -1,3 +1,13 @@
+# Suppress known warnings and reset binlogs on all raft followers.
+--disable_query_log
+connection server_1;
+call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*");
+connection server_4;
+call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*");
+connection server_5;
+call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*");
+--enable_query_log
+
 let $_rpl_server= $rpl_server_count;
 while ($_rpl_server)
 {

diff --git a/mysql-test/suite/rpl_raft/include/raft_3_node.inc b/mysql-test/suite/rpl_raft/include/raft_3_node.inc
@@ -81,27 +81,14 @@ let $dont_decrement_filename= 1;
 --let $rpl_connection_name= master
 --source include/rpl_connection.inc
 
-# Suppress known warnings and reset binlogs on all raft followers.
---disable_query_log
-call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*");
---enable_query_log
-
 # Create connections to server 4 and 5 (these are not in the ring)
 let $rpl_server_number= 4;
 let $rpl_connection_name= server_4;
 source include/rpl_connect.inc;
---disable_query_log
-connection server_4;
-call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*");
---enable_query_log
 
 let $rpl_server_number= 5;
 let $rpl_connection_name= server_5;
 source include/rpl_connect.inc;
---disable_query_log
-connection server_5;
-call mtr.add_suppression(".*using --replicate-same-server-id in conjunction with --log-slave-updates.*");
---enable_query_log
 
 connection server_1;
 show status like 'rpl_raft_role';
@@ -122,5 +109,19 @@ reset master;
 eval set global gtid_purged='$exec_gtid_set';
 --enable_query_log
 
+connection server_4;
+let $exec_gtid_set= query_get_value(SHOW MASTER STATUS, Executed_Gtid_Set, 1);
+--disable_query_log
+reset master;
+eval set global gtid_purged='$exec_gtid_set';
+--enable_query_log
+
+connection server_5;
+let $exec_gtid_set= query_get_value(SHOW MASTER STATUS, Executed_Gtid_Set, 1);
+--disable_query_log
+reset master;
+eval set global gtid_purged='$exec_gtid_set';
+--enable_query_log
+
 # switch back to server_1
 connection server_1;
diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_apply_events.result b/mysql-test/suite/rpl_raft/r/rpl_raft_apply_events.result
@@ -30,13 +30,13 @@ insert into t1 values();
 include/sync_slave_sql_with_master.inc
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-#	#	Previous_gtids	#	#	Gtid_set
+#	#	Previous_gtids	#	#	
 #	#	Metadata	#	#	Prev HLC_TIME: #
 #	#	Gtid	#	#	SET @@SESSION.GTID_NEXT= 'Gtid_set'
-#	#	Metadata	#	#	HLC_TIME: # Raft term: 1 Raft Index: 3
+#	#	Metadata	#	#	HLC_TIME: # Raft term: 1 Raft Index: 2
 #	#	Query	#	#	use `test`; create table t1 (a int primary key auto_increment) engine = innodb
 #	#	Gtid	#	#	SET @@SESSION.GTID_NEXT= 'Gtid_set'
-#	#	Metadata	#	#	HLC_TIME: # Raft term: 1 Raft Index: 4
+#	#	Metadata	#	#	HLC_TIME: # Raft term: 1 Raft Index: 3
 #	#	Query	#	#	BEGIN
 #	#	Table_map	#	#	table_id: # (test.t1)
 #	#	Write_rows	#	#	table_id: # flags: STMT_END_F

diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result b/mysql-test/suite/rpl_raft/r/rpl_raft_auto_purge_apply_logs.result
@@ -135,6 +135,7 @@ count(*)
 21
 show binary logs;
 Log_name	File_size	Encrypted
+apply-logs-13001.000016	#	#
 apply-logs-13001.000017	#	#
 apply-logs-13001.000018	#	#
 apply-logs-13001.000019	#	#
@@ -149,6 +150,7 @@ count(*)
 21
 show binary logs;
 Log_name	File_size	Encrypted
+apply-logs-13002.000016	#	#
 apply-logs-13002.000017	#	#
 apply-logs-13002.000018	#	#
 apply-logs-13002.000019	#	#
@@ -237,6 +239,7 @@ count(*)
 41
 show binary logs;
 Log_name	File_size	Encrypted
+apply-logs-13002.000037	#	#
 apply-logs-13002.000038	#	#
 apply-logs-13002.000039	#	#
 apply-logs-13002.000040	#	#
@@ -325,6 +328,7 @@ count(*)
 61
 show binary logs;
 Log_name	File_size	Encrypted
+apply-logs-13002.000058	#	#
 apply-logs-13002.000059	#	#
 apply-logs-13002.000060	#	#
 apply-logs-13002.000061	#	#

diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_dump_truncation.result b/mysql-test/suite/rpl_raft/r/rpl_raft_dump_truncation.result
@@ -19,10 +19,6 @@ show status like 'rpl_raft_role';
 Variable_name	Value
 Rpl_raft_role	FOLLOWER
 reset master;
-call mtr.add_suppression("Run function 'before_flush' in plugin 'RPL_RAFT' failed");
-call mtr.add_suppression("Error writing file 'binary-logs-13000'.*");
-call mtr.add_suppression("Commit consensus error set for 1 threads in the group");
-call mtr.add_suppression(".*");
 RESET MASTER;
 RESET SLAVE;
 SET @@GLOBAL.ENABLE_RAFT_PLUGIN = 0;
@@ -40,20 +36,29 @@ Note	1759	Sending passwords in plain text without SSL/TLS is extremely insecure.
 Note	1760	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
 START SLAVE;
 create table t1 (a int primary key) engine = innodb;
+insert into t1 values(0);
+insert into t1 values(1);
+insert into t1 values(2);
 include/sync_slave_sql_with_master.inc
 include/sync_slave_sql_with_master.inc
 include/sync_slave_sql_with_master.inc
 include/sync_slave_sql_with_master.inc
 "Stopping followers"
 "Executing trx"
-insert into t1 values(1);
+insert into t1 values(3);
 "Stopping leader"
 "Resuming followers"
 "Waiting for leader election"
 select * from t1;
 a
+0
+1
+2
 select * from t1;
 a
+0
+1
+2
 "Resuming old leader"
 ERROR HY000: Got error 1 - 'Operation not permitted' during COMMIT
 include/raft_promote_to_leader.inc
@@ -63,14 +68,29 @@ include/sync_slave_sql_with_master.inc
 include/sync_slave_sql_with_master.inc
 select * from t1;
 a
+0
+1
+2
 select * from t1;
 a
+0
+1
+2
 select * from t1;
 a
+0
+1
+2
 select * from t1;
 a
+0
+1
+2
 select * from t1;
 a
+0
+1
+2
 drop table t1;
 include/sync_slave_sql_with_master.inc
 include/sync_slave_sql_with_master.inc

diff --git a/mysql-test/suite/rpl_raft/r/rpl_raft_file_rotation_error.result b/mysql-test/suite/rpl_raft/r/rpl_raft_file_rotation_error.result
@@ -19,7 +19,6 @@ show status like 'rpl_raft_role';
 Variable_name	Value
 Rpl_raft_role	FOLLOWER
 reset master;
-call mtr.add_suppression("Failed to rotate binary log");
 show variables like 'read_only';
 Variable_name	Value
 read_only	OFF