Skip to content

Commit

Permalink
MDEV-33898 : Galera test failure on galera.MW-369
Browse files Browse the repository at this point in the history 
Tests using MW-369.inc sometimes hanged after
signaling two debug sync points inside a Galera
library. Replaced Galera library sync point
with server code sync point when possible and
added more wait_conditions to make sure we are
in correct state.

Tests effected: MW-369, MW-402, MDEV-27276, and
mysql-wsrep#332.

Signed-off-by: Julius Goryavsky <julius.goryavsky@mariadb.com>
  • Loading branch information
@janlindstrom @sysprg
janlindstrom authored and sysprg committed Apr 26, 2024
1 parent 288ea9e commit 1532f12
Show file tree
Hide file tree
Showing 10 changed files with 283 additions and 172 deletions.
2 changes: 0 additions & 2 deletions mysql-test/suite/galera/disabled.def
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,4 @@ galera_as_slave_replay : MDEV-32780 galera_as_slave_replay: assertion in the wsr
galera_slave_replay : MDEV-32780 galera_as_slave_replay: assertion in the wsrep::transaction::before_rollback()
galera_sst_mysqldump_with_key : MDEV-32782 galera_sst_mysqldump_with_key test failed
galera_var_ignore_apply_errors : MENT-1997 galera_var_ignore_apply_errors test freezes
MDEV-22232 : temporarily disabled at the request of Codership
MW-402 : temporarily disabled at the request of Codership
galera_desync_overlapped : MDEV-21538 galera_desync_overlapped MTR failed: Result content mismatch
4 changes: 4 additions & 0 deletions mysql-test/suite/galera/galera_4nodes.cnf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#gale
wsrep_node_address='127.0.0.1:@mysqld.1.#galera_port'
wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
auto-increment-offset=1

[mysqld.2]
wsrep-on=1
Expand All @@ -32,6 +33,7 @@ wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#gale
wsrep_node_address='127.0.0.1:@mysqld.2.#galera_port'
wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
auto-increment-offset=2

[mysqld.3]
wsrep-on=1
Expand All @@ -43,6 +45,7 @@ wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.3.#gale
wsrep_node_address='127.0.0.1:@mysqld.3.#galera_port'
wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'
auto-increment-offset=3

[mysqld.4]
wsrep-on=1
Expand All @@ -54,6 +57,7 @@ wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.4.#gale
wsrep_node_address='127.0.0.1:@mysqld.4.#galera_port'
wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port
wsrep_sst_receive_address='127.0.0.1:@mysqld.4.#sst_port'
auto-increment-offset=4

[ENV]
NODE_MYPORT_1= @mysqld.1.port
Expand Down
6 changes: 2 additions & 4 deletions mysql-test/suite/galera/r/MDEV-27276.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,15 @@ connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
connection node_1;
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
connection node_1;
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
connection node_1a;
SET SESSION DEBUG_SYNC = "RESET";
connection node_2;
SELECT * FROM p;
id f2
Expand Down
126 changes: 56 additions & 70 deletions mysql-test/suite/galera/r/MW-369.result
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,20 @@ START TRANSACTION;
DELETE FROM p WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
INSERT INTO c VALUES (1, 1);
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
connection node_2;
Expand All @@ -54,22 +52,20 @@ START TRANSACTION;
UPDATE p SET f2 = 1 WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
UPDATE c SET f2 = 1 WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
connection node_2;
SELECT * FROM p;
Expand All @@ -94,22 +90,20 @@ START TRANSACTION;
UPDATE p SET f2 = 1 WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
DELETE FROM c WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
connection node_2;
SELECT * FROM p;
Expand All @@ -130,22 +124,20 @@ START TRANSACTION;
UPDATE p SET f2 = 1 WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
INSERT INTO c VALUES (1, 0);;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
connection node_2;
Expand All @@ -170,22 +162,20 @@ START TRANSACTION;
DELETE FROM p WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
UPDATE c SET f2 = 1 WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
connection node_2;
Expand Down Expand Up @@ -215,22 +205,20 @@ START TRANSACTION;
INSERT INTO cf (f1, p_id) VALUES (10, 1);
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
INSERT INTO cf (f1, p_id) VALUES (20, 1);
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
connection node_2;
SELECT * FROM pf;
Expand All @@ -255,22 +243,20 @@ START TRANSACTION;
UPDATE pg SET f2 = 1 WHERE f1 = 1;
connection node_1a;
SET SESSION wsrep_sync_wait = 0;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
connection node_2;
INSERT INTO cg VALUES (1, 1, 0);
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
connection node_1;
SET DEBUG_SYNC = "wsrep_after_certification SIGNAL after_certification_reached WAIT_FOR continue_after_certification";
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR after_certification_reached";
SET DEBUG_SYNC = 'now SIGNAL continue_after_certification';
SET DEBUG_SYNC = 'now SIGNAL signal.wsrep_apply_cb';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = 'RESET';
connection node_1;
connection node_2;
SELECT * FROM pg;
Expand Down
Loading

0 comments on commit 1532f12

Please sign in to comment.