Galera MTR Tests: GAL-405 Initial implementation of GCache recovery o…

…n startup.

mysql-test/suite/galera/r/galera_defaults.result

@@ -50,7 +50,7 @@ WSREP_SST_DONOR
 WSREP_SST_DONOR_REJECTS_QUERIES	OFF
 WSREP_SST_METHOD	rsync
 WSREP_SYNC_WAIT	7
-<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <RECV_Q_HARD_LIMIT>;gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992; 
+<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <RECV_Q_HARD_LIMIT>;gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992; 
 SELECT COUNT(*) FROM INFORMATION_SCHEMA.GLOBAL_STATUS
 WHERE VARIABLE_NAME LIKE 'wsrep_%'
 AND VARIABLE_NAME != 'wsrep_debug_sync_waiters';

mysql-test/suite/galera/r/galera_gcache_recover.result

@@ -0,0 +1,18 @@
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+SET SESSION wsrep_sync_wait = 0;
+SET SESSION wsrep_sync_wait = 0;
+Killing server ...
+INSERT INTO t1 VALUES (2);
+Killing server ...
+Performing --wsrep-recover ...
+Using --wsrep-start-position when starting mysqld ...
+INSERT INTO t1 VALUES (3);
+Performing --wsrep-recover ...
+Using --wsrep-start-position when starting mysqld ...
+include/diff_servers.inc [servers=1 2]
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+include/assert_grep.inc [async IST sender starting to serve]
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+include/assert_grep.inc [Recovering GCache ring buffer: found gapless sequence]
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_gcache_recover_full_gcache.result

@@ -0,0 +1,19 @@
+SET SESSION wsrep_sync_wait = 0;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB;
+SET SESSION wsrep_sync_wait = 0;
+Killing server ...
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+Killing server ...
+Performing --wsrep-recover ...
+Using --wsrep-start-position when starting mysqld ...
+Performing --wsrep-recover ...
+Using --wsrep-start-position when starting mysqld ...
+include/diff_servers.inc [servers=1 2]
+DROP TABLE t1;
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+include/assert_grep.inc [IST first seqno 2 not found from cache, falling back to SST]
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");

mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result

@@ -0,0 +1,107 @@
+SET SESSION wsrep_sync_wait = 0;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB;
+CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
+CREATE PROCEDURE insert_simple ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+WHILE 1 DO
+INSERT INTO t1 (f1, f2) VALUES (DEFAULT,'abcdef');
+END WHILE;
+END|
+CREATE PROCEDURE insert_multi ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+WHILE 1 DO
+INSERT INTO t1 (f1) VALUES (DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT);
+END WHILE;
+END|
+CREATE PROCEDURE insert_transaction ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+SET AUTOCOMMIT = OFF;
+WHILE 1 DO
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+INSERT INTO t1 (f1) VALUES (DEFAULT);
+COMMIT;
+END WHILE;
+END|
+CREATE PROCEDURE update_simple ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+WHILE 1 DO
+UPDATE t1 SET f2 = CONCAT(f2,f2);
+END WHILE;
+END|
+CREATE PROCEDURE insert_1k ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+WHILE 1 DO
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024));
+END WHILE;
+END|
+CREATE PROCEDURE insert_1m ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+WHILE 1 DO
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024));
+END WHILE;
+END|
+CREATE PROCEDURE insert_10m ()
+BEGIN
+DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
+SET SESSION wsrep_sync_wait = 0;
+WHILE 1 DO
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+END WHILE;
+END|
+CALL insert_simple();;
+CALL insert_multi();;
+CALL insert_transaction ();;
+CALL update_simple ();;
+CALL insert_1k ();;
+CALL insert_1m ();;
+CALL insert_10m ();;
+SET SESSION wsrep_sync_wait = 0;
+Killing server ...
+Killing server ...
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+ERROR HY000: Lost connection to MySQL server during query
+Performing --wsrep-recover ...
+Using --wsrep-start-position when starting mysqld ...
+Performing --wsrep-recover ...
+Using --wsrep-start-position when starting mysqld ...
+include/diff_servers.inc [servers=1 2]
+DROP TABLE t1;
+DROP TABLE ten;
+DROP PROCEDURE insert_simple;
+DROP PROCEDURE insert_multi;
+DROP PROCEDURE insert_transaction;
+DROP PROCEDURE update_simple;
+DROP PROCEDURE insert_1k;
+DROP PROCEDURE insert_1m;
+CALL mtr.add_suppression("conflict state 7 after post commit");
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+include/assert_grep.inc [async IST sender starting to serve]
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+include/assert_grep.inc [Recovering GCache ring buffer: found gapless sequence]

mysql-test/suite/galera/t/galera_gcache_recover.cnf

@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.recover=yes;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.recover=yes;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_gcache_recover.test

@@ -0,0 +1,77 @@
+#
+# Kill entire cluster while gcache.recover=yes. Expect that node #2 will rejoin using IST
+#
+
+--source include/galera_cluster.inc
+--source include/big_test.inc
+
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+SET SESSION wsrep_sync_wait = 0;
+
+--connection node_2
+--let $wait_condition = SELECT COUNT(*) > 0 FROM t1;
+--source include/wait_condition.inc
+
+SET SESSION wsrep_sync_wait = 0;
+--source include/kill_galera.inc
+
+--connection node_1
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+INSERT INTO t1 VALUES (2);
+
+--source include/kill_galera.inc
+
+--sleep 1
+
+--connection node_1
+--let $galera_wsrep_recover_server_id=1
+--source suite/galera/include/galera_wsrep_recover.inc
+
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--source include/start_mysqld.inc
+
+INSERT INTO t1 VALUES (3);
+
+--connection node_2
+--let $galera_wsrep_recover_server_id=2
+--source suite/galera/include/galera_wsrep_recover.inc
+
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--source include/start_mysqld.inc
+
+--connection node_1
+--source include/wait_until_connected_again.inc
+--source include/galera_wait_ready.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+
+--let $diff_servers = 1 2
+--source include/diff_servers.inc
+
+--connection node_1
+# Warning happens when the cluster is started for the first time
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+
+# Confirm that IST took place
+--let $assert_text = async IST sender starting to serve
+--let $assert_select = async IST sender starting to serve
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let $assert_only_after = starting as process
+--source include/assert_grep.inc
+
+--connection node_2
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+
+# Confirm that gcache recovery took place
+
+--let $assert_text = Recovering GCache ring buffer: found gapless sequence
+--let $assert_select = Recovering GCache ring buffer: found gapless sequence
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
+--let $assert_only_after = starting as process
+--source include/assert_grep.inc
+
+DROP TABLE t1;

mysql-test/suite/galera/t/galera_gcache_recover_full_gcache.cnf

@@ -0,0 +1,9 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+max_allowed_packet=10M
+innodb_log_file_size=110M
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.recover=yes;pc.ignore_sb=true;gcache.size=10M'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.recover=yes;pc.ignore_sb=true;gcache.size=10M'

mysql-test/suite/galera/t/galera_gcache_recover_full_gcache.test

@@ -0,0 +1,59 @@
+#
+# Attempt gcache recovery on a full gcache. Node will not be able to join via IST due to gcache rollover
+#
+
+--source include/galera_cluster.inc
+--source include/big_test.inc
+
+SET SESSION wsrep_sync_wait = 0;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB;
+
+--connection node_2
+SET SESSION wsrep_sync_wait = 0;
+--source include/kill_galera.inc
+
+--connection node_1
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
+--source include/kill_galera.inc
+
+--connection node_1
+--let $galera_wsrep_recover_server_id=1
+--source suite/galera/include/galera_wsrep_recover.inc
+
+--let $_expect_file_name = $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--source include/start_mysqld.inc
+--connection node_2
+--let $galera_wsrep_recover_server_id=2
+--source suite/galera/include/galera_wsrep_recover.inc
+
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
+--source include/start_mysqld.inc
+
+--connection node_1
+--source include/wait_until_connected_again.inc
+--source include/galera_wait_ready.inc
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+
+--let $diff_servers = 1 2
+--source include/diff_servers.inc
+
+--connection node_1
+DROP TABLE t1;
+
+# Warning always happens when the cluster is started for the first time
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
+
+# Confirm that IST did not take place
+--let $assert_text = IST first seqno 2 not found from cache, falling back to SST
+--let $assert_select = IST first seqno 2 not found from cache, falling back to SST
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let $assert_only_after = starting as process
+--source include/assert_grep.inc
+
+--connection node_2
+CALL mtr.add_suppression("Skipped GCache ring buffer recovery");

mysql-test/suite/galera/t/galera_gcache_recover_manytrx.cnf

@@ -0,0 +1,8 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+innodb_log_file_size=110M
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.recover=yes;pc.ignore_sb=true;'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.recover=yes;pc.ignore_sb=true;'