Skip to content

Commit

Permalink
MDEV-23851 BF-BF Conflict issue because of UK GAP locks
Browse files Browse the repository at this point in the history 
Add missing dbug sync point and correct the test case.
  • Loading branch information
Jan Lindström committed Mar 17, 2021
1 parent f4e4bff commit 5511c21
Show file tree
Hide file tree
Showing 3 changed files with 218 additions and 37 deletions.
74 changes: 58 additions & 16 deletions mysql-test/suite/galera/r/galera_UK_conflict.result
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
connection node_2;
connection node_1;
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int, unique key keyj (f2));
INSERT INTO t1 VALUES (1, 1, 0);
INSERT INTO t1 VALUES (3, 3, 0);
Expand All @@ -23,14 +25,14 @@ connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
connection node_1;
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
Expand All @@ -39,13 +41,13 @@ SET debug_sync='RESET';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET GLOBAL debug_dbug = NULL;
SET debug_sync='RESET';
SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_enter_sync';
SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
connection node_1;
SELECT * FROM t1;
Expand All @@ -55,8 +57,6 @@ f1 f2 f3
4 4 2
5 5 2
10 10 0
wsrep_local_replays
1
SET GLOBAL wsrep_slave_threads = DEFAULT;
connection node_2;
SELECT * FROM t1;
Expand All @@ -68,22 +68,64 @@ f1 f2 f3
10 10 0
INSERT INTO t1 VALUES (7,7,7);
INSERT INTO t1 VALUES (8,8,8);
SELECT * FROM t1;
f1 f2 f3
1 1 0
3 3 1
4 4 2
5 5 2
7 7 7
8 8 8
10 10 0
DROP TABLE t1;
test scenario 2
connection node_1;
CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int, unique key keyj (f2));
INSERT INTO t1 VALUES (1, 1, 0);
INSERT INTO t1 VALUES (3, 3, 0);
INSERT INTO t1 VALUES (10, 10, 0);
SET GLOBAL wsrep_slave_threads = 3;
SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
connection node_1;
SET SESSION wsrep_sync_wait=0;
START TRANSACTION;
DELETE FROM t1 WHERE f2 = 3;
INSERT INTO t1 VALUES (3, 3, 1);
connection node_1a;
SET SESSION wsrep_sync_wait=0;
connection node_2;
INSERT INTO t1 VALUES (5, 5, 2);
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
connection node_1;
COMMIT;
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_replay_cb";
SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_replay_cb_reached";
SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync';
connection node_2;
INSERT INTO t1 VALUES (4, 4, 2);
connection node_1a;
SET SESSION wsrep_on = 0;
SET SESSION wsrep_on = 1;
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync';
SET GLOBAL wsrep_provider_options = 'dbug=';
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_replay_cb";
SET GLOBAL debug_dbug = NULL;
SET debug_sync='RESET';
connection node_1;
SET GLOBAL wsrep_slave_threads = DEFAULT;
connection node_2;
SELECT * FROM t1;
f1 f2 f3
1 1 0
3 3 1
4 4 2
5 5 2
7 7 7
10 10 0
INSERT INTO t1 VALUES (7,7,7);
INSERT INTO t1 VALUES (8,8,8);
DROP TABLE t1;
170 changes: 149 additions & 21 deletions mysql-test/suite/galera/t/galera_UK_conflict.test
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,31 @@
#
# This test tests the operation of transaction replay with a scenario
# where two subsequent write sets in applying conflict with local transaction
# where two subsequent write sets being applied conflict with local transaction
# in commit phase. The conflict is "false positive" confict on GAP lock in
# secondary unique index.
# The first applier will cause BF abort for the local committer, which
# starts replaying because of positive certification.
# In buggy version, scenatio continues so that ehile the local transaction
# In buggy version, the test scenario continues so that while the local transaction
# is replaying, the latter applier experiences similar UK GAP lock conflict
# and forces the replayer to abort second time.
# In fixed version, this latter BF abort should not happen.
# In fixed version, this latter replayer BF abort should not happen.
#

--source include/galera_cluster.inc
--source include/have_innodb.inc
--source include/have_debug_sync.inc
--source include/galera_have_debug_sync.inc

--let $wsrep_local_replays_old = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`

--let $expected_wsrep_local_replays = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`

CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int, unique key keyj (f2));
INSERT INTO t1 VALUES (1, 1, 0);
INSERT INTO t1 VALUES (3, 3, 0);
INSERT INTO t1 VALUES (10, 10, 0);

# we will need 2 appliers threads for applyin two write sets in parallel in node1
# and 1 applier thread for handling replaying
# we will need 2 appliers threads for applyin two writes ets in parallel in node1
# and 1 applier thread for handling replaying
SET GLOBAL wsrep_slave_threads = 3;
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";

Expand All @@ -42,7 +43,7 @@ INSERT INTO t1 VALUES (3, 3, 1);
--connection node_1a
SET SESSION wsrep_sync_wait=0;

# send from node 2 first INSERT transaction, which will conflict on GAP lock in node 1
# send from node 2 first an INSERT transaction, which will conflict on GAP lock in node 1
--connection node_2
INSERT INTO t1 VALUES (5, 5, 2);

Expand All @@ -67,20 +68,20 @@ INSERT INTO t1 VALUES (4, 4, 2);
# both appliers are now waiting in separate sync points

# Block the local commit, send the COMMIT and wait until it gets blocked
--let $galera_sync_point = commit_monitor_enter_sync
--let $galera_sync_point = commit_monitor_master_enter_sync
--source include/galera_set_sync_point.inc

--connection node_1
--send COMMIT

--connection node_1a
# wait for the local commit to enter in commit monitor wait state
--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_enter_sync
--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync
--source include/galera_wait_sync_point.inc
--source include/galera_clear_sync_point.inc

# release the local transaction to continue with commit
--let $galera_sync_point = commit_monitor_enter_sync
--let $galera_sync_point = commit_monitor_master_enter_sync
--source include/galera_signal_sync_point.inc
--source include/galera_clear_sync_point.inc

Expand All @@ -90,18 +91,23 @@ SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET GLOBAL debug_dbug = NULL;
SET debug_sync='RESET';

# wait for BF abort to happen and replaying begin
--let $wait_condition = SELECT VARIABLE_VALUE= $expected_wsrep_local_replays FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
--let $wait_condition_on_error_output= SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
--source include/wait_condition_with_debug.inc

# set another sync point for second applier
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";

# letting the second appier to move forward
--let $galera_sync_point = apply_monitor_slave_enter_sync
--source include/galera_signal_sync_point.inc

# waiting until second applier is in wait
# waiting until second applier is in wait state
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";

# stopping second applier before commit
--let $galera_sync_point = commit_monitor_enter_sync
--let $galera_sync_point = commit_monitor_slave_enter_sync
--source include/galera_set_sync_point.inc
--source include/galera_clear_sync_point.inc

Expand All @@ -112,9 +118,9 @@ SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
SET GLOBAL debug_dbug = NULL;
SET debug_sync='RESET';

# with fixed version, second applier has reached commit monitor, and we can
# with fixed version, second applier has reached comit monitor, and we can
# release it to complete
--let $galera_sync_point = commit_monitor_enter_sync
--let $galera_sync_point = commit_monitor_slave_enter_sync
--source include/galera_signal_sync_point.inc
--source include/galera_clear_sync_point.inc

Expand All @@ -124,12 +130,6 @@ SET debug_sync='RESET';

SELECT * FROM t1;

# wsrep_local_replays has increased by 1
--let $wsrep_local_replays_new = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
--disable_query_log
--eval SELECT $wsrep_local_replays_new - $wsrep_local_replays_old = 1 AS wsrep_local_replays;
--enable_query_log

# returning original slave thread count
SET GLOBAL wsrep_slave_threads = DEFAULT;

Expand All @@ -140,9 +140,137 @@ SELECT * FROM t1;
# original state in node 1
INSERT INTO t1 VALUES (7,7,7);
INSERT INTO t1 VALUES (8,8,8);
SELECT * FROM t1;

DROP TABLE t1;

##################################################################################
# test scenario 2
#
# commit order is now: INSERT-1, local COMMIT, INSERT-2
# while local trx is replaying, the latter applier has applied and is waiting
# for commit.
# The point in this scenario is to verify that replayer does not try to abort
# the latter applier
#################################################################################

--echo test scenario 2

--connection node_1
--let $expected_wsrep_local_replays = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`

CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int, unique key keyj (f2));
INSERT INTO t1 VALUES (1, 1, 0);
INSERT INTO t1 VALUES (3, 3, 0);
INSERT INTO t1 VALUES (10, 10, 0);

# we will need 2 appliers threads for applyin two writes sets in parallel in node1
# and 1 applier thread for handling replaying
SET GLOBAL wsrep_slave_threads = 3;

# set sync point for the first INSERT applier
--let $galera_sync_point = apply_monitor_slave_enter_sync
--source include/galera_set_sync_point.inc

--connection node_1
# starting a transaction, which deletes and inserts the middle row in test table
# this will be victim of false positive conflict with appliers
SET SESSION wsrep_sync_wait=0;
START TRANSACTION;

DELETE FROM t1 WHERE f2 = 3;
INSERT INTO t1 VALUES (3, 3, 1);

# Control connection to manage sync points for appliers
--connection node_1a
SET SESSION wsrep_sync_wait=0;

# send from node 2 first an INSERT transaction, which will conflict on GAP lock in node 1
--connection node_2
INSERT INTO t1 VALUES (5, 5, 2);

--connection node_1a
# wait to see the INSERT in apply_cb sync point
--let $galera_sync_point = apply_monitor_slave_enter_sync
--source include/galera_wait_sync_point.inc
--source include/galera_clear_sync_point.inc

# Block the local commit, send the COMMIT and wait until it gets blocked
--let $galera_sync_point = commit_monitor_master_enter_sync
--source include/galera_set_sync_point.inc

--connection node_1
--send COMMIT

--connection node_1a
# wait for the local commit to enter in commit monitor wait state
--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync
--source include/galera_wait_sync_point.inc
--source include/galera_clear_sync_point.inc

# set sync point before replaying
SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_replay_cb";

# release the local transaction to continue with commit
# it should advance and end up waiting in commit monitor for his turn
--let $galera_sync_point = commit_monitor_master_enter_sync
--source include/galera_signal_sync_point.inc
--source include/galera_clear_sync_point.inc

# and now release the first applier, it should force local trx to abort
--let $galera_sync_point = apply_monitor_slave_enter_sync
--source include/galera_signal_sync_point.inc
--source include/galera_clear_sync_point.inc

# wait for BF abort to happen and replaying begin
--let $wait_condition = SELECT VARIABLE_VALUE= $expected_wsrep_local_replays FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
--let $wait_condition_on_error_output= SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
--source include/wait_condition_with_debug.inc

# replayer should now be in stopped in sync point
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_replay_cb_reached";

# set sync point for the second INSERT
--let $galera_sync_point = commit_monitor_slave_enter_sync
--source include/galera_set_sync_point.inc

--connection node_2
# send second insert into same GAP in test table
INSERT INTO t1 VALUES (4, 4, 2);

--connection node_1a
# wait for the second applier to enter in commit monitor wait state
--let $galera_sync_point = commit_monitor_slave_enter_sync
--source include/galera_wait_sync_point.inc
--source include/galera_clear_sync_point.inc

# and, letting the second appier to move forward, it will stop naturally
# to wait for commit order after replayer's commit
--let $galera_sync_point = commit_monitor_slave_enter_sync
--source include/galera_signal_sync_point.inc
--source include/galera_clear_sync_point.inc

# and now release the replayer, if all is good,it will commit before the second applier
SET GLOBAL DEBUG_DBUG = "";
SET DEBUG_SYNC = "now SIGNAL signal.wsrep_replay_cb";
SET GLOBAL debug_dbug = NULL;
SET debug_sync='RESET';

# local commit should succeed
--connection node_1
--reap

--let $wait_condition = SELECT COUNT(*)=5 FROM t1;
--source include/wait_condition.inc

# returning original slave thread count
SET GLOBAL wsrep_slave_threads = DEFAULT;

--connection node_2
SELECT * FROM t1;

# replicate some transactions, so that wsrep slave thread count can reach
# original state in node 1
INSERT INTO t1 VALUES (7,7,7);
INSERT INTO t1 VALUES (8,8,8);

DROP TABLE t1;
Loading

0 comments on commit 5511c21

Please sign in to comment.