Skip to content

Commit 108fd77

Browse files
philip-galeraSachinSetiya
philip-galera
authored and
SachinSetiya
committed
Galera MTR Tests: GAL-405 Initial implementation of GCache recovery on startup.
1 parent 9be994b commit 108fd77

10 files changed

+521
-1
lines changed

‎mysql-test/suite/galera/r/galera_defaults.result

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ WSREP_SST_DONOR
4747
WSREP_SST_DONOR_REJECTS_QUERIES OFF
4848
WSREP_SST_METHOD rsync
4949
WSREP_SYNC_WAIT 7
50-
<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <RECV_Q_HARD_LIMIT>;gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;
50+
<BASE_DIR>; <BASE_HOST>; <BASE_PORT>; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT30S; evs.info_log_mask = 0; evs.install_timeout = PT15S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT10S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; <GCACHE_DIR>; gcache.keep_pages_size = 0; gcache.mem_size = 0; <GCACHE_NAME>; gcache.page_size = 128M; gcache.recover = no; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; <RECV_Q_HARD_LIMIT>;gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; <GMCAST_LISTEN_ADDR>; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; <IST_RECV_ADDR>; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT90S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;
5151
SELECT COUNT(*) FROM INFORMATION_SCHEMA.GLOBAL_STATUS
5252
WHERE VARIABLE_NAME LIKE 'wsrep_%'
5353
AND VARIABLE_NAME != 'wsrep_debug_sync_waiters';

‎mysql-test/suite/galera/r/galera_gcache_recover.result

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
2+
INSERT INTO t1 VALUES (1);
3+
SET SESSION wsrep_sync_wait = 0;
4+
SET SESSION wsrep_sync_wait = 0;
5+
Killing server ...
6+
INSERT INTO t1 VALUES (2);
7+
Killing server ...
8+
Performing --wsrep-recover ...
9+
Using --wsrep-start-position when starting mysqld ...
10+
INSERT INTO t1 VALUES (3);
11+
Performing --wsrep-recover ...
12+
Using --wsrep-start-position when starting mysqld ...
13+
include/diff_servers.inc [servers=1 2]
14+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
15+
include/assert_grep.inc [async IST sender starting to serve]
16+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
17+
include/assert_grep.inc [Recovering GCache ring buffer: found gapless sequence]
18+
DROP TABLE t1;

‎mysql-test/suite/galera/r/galera_gcache_recover_full_gcache.result

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
SET SESSION wsrep_sync_wait = 0;
2+
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB;
3+
SET SESSION wsrep_sync_wait = 0;
4+
Killing server ...
5+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
6+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
7+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
8+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
9+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
10+
Killing server ...
11+
Performing --wsrep-recover ...
12+
Using --wsrep-start-position when starting mysqld ...
13+
Performing --wsrep-recover ...
14+
Using --wsrep-start-position when starting mysqld ...
15+
include/diff_servers.inc [servers=1 2]
16+
DROP TABLE t1;
17+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
18+
include/assert_grep.inc [IST first seqno 2 not found from cache, falling back to SST]
19+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");

‎mysql-test/suite/galera/r/galera_gcache_recover_manytrx.result

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
SET SESSION wsrep_sync_wait = 0;
2+
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB;
3+
CREATE TABLE ten (f1 INTEGER) ENGINE=InnoDB;
4+
INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
5+
CREATE PROCEDURE insert_simple ()
6+
BEGIN
7+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
8+
SET SESSION wsrep_sync_wait = 0;
9+
WHILE 1 DO
10+
INSERT INTO t1 (f1, f2) VALUES (DEFAULT,'abcdef');
11+
END WHILE;
12+
END|
13+
CREATE PROCEDURE insert_multi ()
14+
BEGIN
15+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
16+
SET SESSION wsrep_sync_wait = 0;
17+
WHILE 1 DO
18+
INSERT INTO t1 (f1) VALUES (DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT),(DEFAULT);
19+
END WHILE;
20+
END|
21+
CREATE PROCEDURE insert_transaction ()
22+
BEGIN
23+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
24+
SET SESSION wsrep_sync_wait = 0;
25+
SET AUTOCOMMIT = OFF;
26+
WHILE 1 DO
27+
START TRANSACTION;
28+
INSERT INTO t1 (f1) VALUES (DEFAULT);
29+
INSERT INTO t1 (f1) VALUES (DEFAULT);
30+
INSERT INTO t1 (f1) VALUES (DEFAULT);
31+
INSERT INTO t1 (f1) VALUES (DEFAULT);
32+
INSERT INTO t1 (f1) VALUES (DEFAULT);
33+
INSERT INTO t1 (f1) VALUES (DEFAULT);
34+
INSERT INTO t1 (f1) VALUES (DEFAULT);
35+
INSERT INTO t1 (f1) VALUES (DEFAULT);
36+
INSERT INTO t1 (f1) VALUES (DEFAULT);
37+
INSERT INTO t1 (f1) VALUES (DEFAULT);
38+
COMMIT;
39+
END WHILE;
40+
END|
41+
CREATE PROCEDURE update_simple ()
42+
BEGIN
43+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
44+
SET SESSION wsrep_sync_wait = 0;
45+
WHILE 1 DO
46+
UPDATE t1 SET f2 = CONCAT(f2,f2);
47+
END WHILE;
48+
END|
49+
CREATE PROCEDURE insert_1k ()
50+
BEGIN
51+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
52+
SET SESSION wsrep_sync_wait = 0;
53+
WHILE 1 DO
54+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024));
55+
END WHILE;
56+
END|
57+
CREATE PROCEDURE insert_1m ()
58+
BEGIN
59+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
60+
SET SESSION wsrep_sync_wait = 0;
61+
WHILE 1 DO
62+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024));
63+
END WHILE;
64+
END|
65+
CREATE PROCEDURE insert_10m ()
66+
BEGIN
67+
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
68+
SET SESSION wsrep_sync_wait = 0;
69+
WHILE 1 DO
70+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
71+
END WHILE;
72+
END|
73+
CALL insert_simple();;
74+
CALL insert_multi();;
75+
CALL insert_transaction ();;
76+
CALL update_simple ();;
77+
CALL insert_1k ();;
78+
CALL insert_1m ();;
79+
CALL insert_10m ();;
80+
SET SESSION wsrep_sync_wait = 0;
81+
Killing server ...
82+
Killing server ...
83+
ERROR HY000: Lost connection to MySQL server during query
84+
ERROR HY000: Lost connection to MySQL server during query
85+
ERROR HY000: Lost connection to MySQL server during query
86+
ERROR HY000: Lost connection to MySQL server during query
87+
ERROR HY000: Lost connection to MySQL server during query
88+
ERROR HY000: Lost connection to MySQL server during query
89+
ERROR HY000: Lost connection to MySQL server during query
90+
Performing --wsrep-recover ...
91+
Using --wsrep-start-position when starting mysqld ...
92+
Performing --wsrep-recover ...
93+
Using --wsrep-start-position when starting mysqld ...
94+
include/diff_servers.inc [servers=1 2]
95+
DROP TABLE t1;
96+
DROP TABLE ten;
97+
DROP PROCEDURE insert_simple;
98+
DROP PROCEDURE insert_multi;
99+
DROP PROCEDURE insert_transaction;
100+
DROP PROCEDURE update_simple;
101+
DROP PROCEDURE insert_1k;
102+
DROP PROCEDURE insert_1m;
103+
CALL mtr.add_suppression("conflict state 7 after post commit");
104+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
105+
include/assert_grep.inc [async IST sender starting to serve]
106+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
107+
include/assert_grep.inc [Recovering GCache ring buffer: found gapless sequence]

‎mysql-test/suite/galera/t/galera_gcache_recover.cnf

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
!include ../galera_2nodes.cnf
2+
3+
[mysqld.1]
4+
wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.recover=yes;pc.ignore_sb=true'
5+
6+
[mysqld.2]
7+
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.recover=yes;pc.ignore_sb=true'

‎mysql-test/suite/galera/t/galera_gcache_recover.test

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#
2+
# Kill entire cluster while gcache.recover=yes. Expect that node #2 will rejoin using IST
3+
#
4+
5+
--source include/galera_cluster.inc
6+
--source include/big_test.inc
7+
8+
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
9+
INSERT INTO t1 VALUES (1);
10+
SET SESSION wsrep_sync_wait = 0;
11+
12+
--connection node_2
13+
--let $wait_condition = SELECT COUNT(*) > 0 FROM t1;
14+
--source include/wait_condition.inc
15+
16+
SET SESSION wsrep_sync_wait = 0;
17+
--source include/kill_galera.inc
18+
19+
--connection node_1
20+
--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
21+
--source include/wait_condition.inc
22+
23+
INSERT INTO t1 VALUES (2);
24+
25+
--source include/kill_galera.inc
26+
27+
--sleep 1
28+
29+
--connection node_1
30+
--let $galera_wsrep_recover_server_id=1
31+
--source suite/galera/include/galera_wsrep_recover.inc
32+
33+
--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
34+
--source include/start_mysqld.inc
35+
36+
INSERT INTO t1 VALUES (3);
37+
38+
--connection node_2
39+
--let $galera_wsrep_recover_server_id=2
40+
--source suite/galera/include/galera_wsrep_recover.inc
41+
42+
--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
43+
--source include/start_mysqld.inc
44+
45+
--connection node_1
46+
--source include/wait_until_connected_again.inc
47+
--source include/galera_wait_ready.inc
48+
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
49+
50+
--let $diff_servers = 1 2
51+
--source include/diff_servers.inc
52+
53+
--connection node_1
54+
# Warning happens when the cluster is started for the first time
55+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
56+
57+
# Confirm that IST took place
58+
--let $assert_text = async IST sender starting to serve
59+
--let $assert_select = async IST sender starting to serve
60+
--let $assert_count = 1
61+
--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
62+
--let $assert_only_after = starting as process
63+
--source include/assert_grep.inc
64+
65+
--connection node_2
66+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
67+
68+
# Confirm that gcache recovery took place
69+
70+
--let $assert_text = Recovering GCache ring buffer: found gapless sequence
71+
--let $assert_select = Recovering GCache ring buffer: found gapless sequence
72+
--let $assert_count = 1
73+
--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
74+
--let $assert_only_after = starting as process
75+
--source include/assert_grep.inc
76+
77+
DROP TABLE t1;

‎mysql-test/suite/galera/t/galera_gcache_recover_full_gcache.cnf

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
!include ../galera_2nodes.cnf
2+
3+
[mysqld.1]
4+
max_allowed_packet=10M
5+
innodb_log_file_size=110M
6+
wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.recover=yes;pc.ignore_sb=true;gcache.size=10M'
7+
8+
[mysqld.2]
9+
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.recover=yes;pc.ignore_sb=true;gcache.size=10M'

‎mysql-test/suite/galera/t/galera_gcache_recover_full_gcache.test

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#
2+
# Attempt gcache recovery on a full gcache. Node will not be able to join via IST due to gcache rollover
3+
#
4+
5+
--source include/galera_cluster.inc
6+
--source include/big_test.inc
7+
8+
SET SESSION wsrep_sync_wait = 0;
9+
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 LONGBLOB) ENGINE=InnoDB;
10+
11+
--connection node_2
12+
SET SESSION wsrep_sync_wait = 0;
13+
--source include/kill_galera.inc
14+
15+
--connection node_1
16+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
17+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
18+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
19+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
20+
INSERT INTO t1 (f2) VALUES (REPEAT('x', 1024 * 1024 * 10));
21+
--source include/kill_galera.inc
22+
23+
--connection node_1
24+
--let $galera_wsrep_recover_server_id=1
25+
--source suite/galera/include/galera_wsrep_recover.inc
26+
27+
--let $_expect_file_name = $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
28+
--source include/start_mysqld.inc
29+
--connection node_2
30+
--let $galera_wsrep_recover_server_id=2
31+
--source suite/galera/include/galera_wsrep_recover.inc
32+
33+
--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect
34+
--source include/start_mysqld.inc
35+
36+
--connection node_1
37+
--source include/wait_until_connected_again.inc
38+
--source include/galera_wait_ready.inc
39+
--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
40+
41+
--let $diff_servers = 1 2
42+
--source include/diff_servers.inc
43+
44+
--connection node_1
45+
DROP TABLE t1;
46+
47+
# Warning always happens when the cluster is started for the first time
48+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");
49+
50+
# Confirm that IST did not take place
51+
--let $assert_text = IST first seqno 2 not found from cache, falling back to SST
52+
--let $assert_select = IST first seqno 2 not found from cache, falling back to SST
53+
--let $assert_count = 1
54+
--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
55+
--let $assert_only_after = starting as process
56+
--source include/assert_grep.inc
57+
58+
--connection node_2
59+
CALL mtr.add_suppression("Skipped GCache ring buffer recovery");

‎mysql-test/suite/galera/t/galera_gcache_recover_manytrx.cnf

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
!include ../galera_2nodes.cnf
2+
3+
[mysqld.1]
4+
innodb_log_file_size=110M
5+
wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.recover=yes;pc.ignore_sb=true;'
6+
7+
[mysqld.2]
8+
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.recover=yes;pc.ignore_sb=true;'

0 commit comments

Comments
 (0)