MDEV-37209 : mtr galera_3nodes.galera_garbd_backup test failures

This contains test case changes only.

1) Make sure InnoDB buffer pool has no dirty pages at begining
2) Use FTWRL to ensure tables are flushed to disk
3) Add wait condition to ensure tables and rows are replicated in the cluster
4) Use sort to ensure same order of the files in the find

Author: janlindstrom

mysql-test/suite/galera_3nodes/r/galera_garbd_backup.result

@@ -4,25 +4,35 @@ connection node_1;
 connection node_2;
 connection node_3;
 connection node_1;
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
 SET GLOBAL innodb_max_dirty_pages_pct=99;
 SET GLOBAL innodb_max_dirty_pages_pct_lwm=99;
-connection node_1;
 CREATE TABLE t1 (f1 INTEGER, f2 varchar(1024)) Engine=InnoDB;
 CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 INSERT INTO t1 (f2) SELECT REPEAT('x', 1024) FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;
+FLUSH TABLES WITH READ LOCK;
+UNLOCK TABLES;
+connection node_2;
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+10000
+SELECT COUNT(*) FROM ten;
+COUNT(*)
+10
 Killing node #3 to free ports for garbd ...
 connection node_3;
 connection node_1;
 SET GLOBAL debug_dbug = "+d,sync.wsrep_donor_state";
 Starting garbd ...
 SET SESSION debug_sync = "now WAIT_FOR sync.wsrep_donor_state_reached";
-SET GLOBAL innodb_max_dirty_pages_pct_lwm=0;
-SET GLOBAL innodb_max_dirty_pages_pct=0;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
 SET SESSION debug_sync = "now SIGNAL signal.wsrep_donor_state";
 SET GLOBAL debug_dbug = "";
 SET debug_sync='RESET';
-connection node_2;
+connection node_1;
 Killing garbd ...
 connection node_1;
 connection node_2;
@@ -31,9 +41,3 @@ DROP TABLE ten;
 Restarting node #3 to satisfy MTR's end-of-test checks
 connection node_3;
 connection node_1;
-connection node_1;
-CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender 1\\.0( \\(.*\\))? is not in state transfer \\(SYNCED\\)\\. Message ignored\\.");
-connection node_2;
-CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender 1\\.0( \\(.*\\))? is not in state transfer \\(SYNCED\\)\\. Message ignored\\.");
-connection node_3;
-CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender 1\\.0( \\(.*\\))? is not in state transfer \\(SYNCED\\)\\. Message ignored\\.");

mysql-test/suite/galera_3nodes/t/galera_garbd_backup.test

@@ -32,14 +32,34 @@
 --let $innodb_max_dirty_pages_pct = `SELECT @@innodb_max_dirty_pages_pct`
 --let $innodb_max_dirty_pages_pct_lwm = `SELECT @@innodb_max_dirty_pages_pct_lwm`
 
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
+
+--let $wait_condition = SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'
+--source include/wait_condition.inc
+
 SET GLOBAL innodb_max_dirty_pages_pct=99;
 SET GLOBAL innodb_max_dirty_pages_pct_lwm=99;
 
---connection node_1
 CREATE TABLE t1 (f1 INTEGER, f2 varchar(1024)) Engine=InnoDB;
 CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 INSERT INTO t1 (f2) SELECT REPEAT('x', 1024) FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4;
+FLUSH TABLES WITH READ LOCK;
+UNLOCK TABLES;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'ten'
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 10 FROM ten
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 10000 FROM t1
+--source include/wait_condition.inc
+
+SELECT COUNT(*) FROM t1;
+SELECT COUNT(*) FROM ten;
 
 --echo Killing node #3 to free ports for garbd ...
 --connection node_3
@@ -60,24 +80,20 @@ SET SESSION debug_sync = "now WAIT_FOR sync.wsrep_donor_state_reached";
 #
 # get hash of data directory contents before BP dirty page flushing
 #
---exec find $datadir -type f ! -name tables_flushed ! -name backup_sst_complete -exec md5sum {} \; | md5sum >$MYSQLTEST_VARDIR/tmp/innodb_before
+--exec find $datadir -type f ! -name tables_flushed ! -name backup_sst_complete -print 0 -exec md5sum {} \; | sort -u | md5sum >$MYSQLTEST_VARDIR/tmp/innodb_before
 
 # this should force buffer pool flushing, if not already done by donor state change transfer
-SET GLOBAL innodb_max_dirty_pages_pct_lwm=0;
-SET GLOBAL innodb_max_dirty_pages_pct=0;
+SET GLOBAL innodb_max_dirty_pages_pct_lwm=0.0;
+SET GLOBAL innodb_max_dirty_pages_pct=0.0;
 
---disable_query_log
---disable_result_log
-select f1 from t1;
-select * from ten;
---enable_result_log
---enable_query_log
+--let $wait_condition = SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'INNODB_BUFFER_POOL_PAGES_DIRTY'
+--source include/wait_condition.inc
 
 #
 #
 # record the hash of data directory contents after BP dirty page flushing
 #
---exec find $datadir -type f ! -name tables_flushed ! -name backup_sst_complete -exec md5sum {} \; | md5sum >$MYSQLTEST_VARDIR/tmp/innodb_after
+--exec find $datadir -type f ! -name tables_flushed ! -name backup_sst_complete -print 0 -exec md5sum {} \; | sort -u | md5sum >$MYSQLTEST_VARDIR/tmp/innodb_after
 
 # there should be no disk writes
 --diff_files $MYSQLTEST_VARDIR/tmp/innodb_before $MYSQLTEST_VARDIR/tmp/innodb_after
@@ -86,7 +102,7 @@ SET SESSION debug_sync = "now SIGNAL signal.wsrep_donor_state";
 SET GLOBAL debug_dbug = "";
 SET debug_sync='RESET';
 
---connection node_2
+--connection node_1
 
 #
 # garbd will die automatically, because of the backup SST script
@@ -122,13 +138,3 @@ let $restart_noprint=2;
 # Restore original auto_increment_offset values.
 --source ../galera/include/auto_increment_offset_restore.inc
 
-# Workaround for galera#101
-
---connection node_1
-CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender 1\\.0( \\(.*\\))? is not in state transfer \\(SYNCED\\)\\. Message ignored\\.");
-
---connection node_2
-CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender 1\\.0( \\(.*\\))? is not in state transfer \\(SYNCED\\)\\. Message ignored\\.");
-
---connection node_3
-CALL mtr.add_suppression("WSREP: Protocol violation\\. JOIN message sender 1\\.0( \\(.*\\))? is not in state transfer \\(SYNCED\\)\\. Message ignored\\.");