MDEV-37141 DML committed within XA transaction block after deadlock error and implicit rollback

Issue: When XA transaction is implicitly rolled back, we keep XA state
XA_ACTIVE and set rm_error to ER_LOCK_DEADLOCK. Other than XA command
we don't check for rm_error and DML and query are executed with a new
transaction.

Fix: One way to fix this issue is to set the XA state to XA_ROLLBACK_ONLY
which is checked while opening table open_tables() and ER_XAER_RMFAIL is
returned for any DML or Query.

Author: mariadb-DebarunBanerjee

mysql-test/main/xa.result

@@ -135,11 +135,13 @@ connection con2;
 update t1 set c = 'aa' where a = 1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 select count(*) from t1;
-count(*)
-2
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ROLLBACK ONLY state
 xa end 'a','c';
-ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ROLLBACK ONLY state
 xa rollback 'a','c';
+select count(*) from t1;
+count(*)
+2
 disconnect con2;
 connect  con3,localhost,root,,;
 connection con3;
@@ -207,7 +209,7 @@ connection default;
 UPDATE t1 SET a=5 WHERE a=1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 XA END 'xid1';
-ERROR XA102: XA_RBDEADLOCK: Transaction branch was rolled back: deadlock was detected
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ROLLBACK ONLY state
 XA ROLLBACK 'xid1';
 XA START 'xid1';
 XA END 'xid1';

mysql-test/main/xa.test

@@ -165,10 +165,12 @@ update t1 set c = 'bb' where a = 2;
 --sleep 1
 --error ER_LOCK_DEADLOCK
 update t1 set c = 'aa' where a = 1;
+--error ER_XAER_RMFAIL
 select count(*) from t1;
---error ER_XA_RBDEADLOCK
+--error ER_XAER_RMFAIL
 xa end 'a','c';
 xa rollback 'a','c';
+select count(*) from t1;
 --disconnect con2
 
 connect (con3,localhost,root,,);
@@ -274,7 +276,7 @@ WHERE ID=$conn_id AND STATE='Searching rows for update';
 
 --error ER_LOCK_DEADLOCK
 UPDATE t1 SET a=5 WHERE a=1;
---error ER_XA_RBDEADLOCK
+--error ER_XAER_RMFAIL
 XA END 'xid1';
 XA ROLLBACK 'xid1';
 

mysql-test/suite/innodb/r/trx_deadlock.result

@@ -35,9 +35,48 @@ SELECT @@in_transaction;
 @@in_transaction
 0
 ROLLBACK;
-disconnect con1;
 SELECT * FROM t1;
 col1	col2
 1	10
 2	100
 DROP TABLE t1;
+#
+# MDEV-37141 DML committed within XA transaction block after deadlock error and implicit rollback
+#
+CREATE TABLE t1(col1 INT PRIMARY KEY, col2 INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 1), (2, 2);
+SELECT * FROM t1;
+col1	col2
+1	1
+2	2
+connection con1;
+XA BEGIN 'x1';
+# XA Trx-1: Lock 1st record
+UPDATE t1 SET col2=10 where col1=1;
+connection default;
+XA BEGIN 'x2';
+# XA Trx-2: Lock 2nd record
+UPDATE t1 SET col2=100 where col1=2;
+connection con1;
+# XA Trx-1: Try locking 1st record : Wait
+UPDATE t1 SET col2=10 where col1=2;
+connection default;
+# Wait for XA Trx-1 to get into lock wait stage
+# XA Trx-2: Try locking 2nd record : Deadlock
+UPDATE t1 SET col2=100 where col1=1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+INSERT INTO t1 VALUES (3, 3), (4, 4);
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ROLLBACK ONLY state
+XA END 'x2';
+ERROR XAE07: XAER_RMFAIL: The command cannot be executed when global transaction is in the  ROLLBACK ONLY state
+XA ROLLBACK 'x2';
+connection con1;
+XA END 'x1';
+XA ROLLBACK 'x1';
+connection default;
+SELECT * FROM t1;
+col1	col2
+1	1
+2	2
+DROP TABLE t1;
+disconnect con1;

mysql-test/suite/innodb/t/trx_deadlock.test

@@ -2,6 +2,7 @@
 --echo # MDEV-36959 Deadlock does not rollback transaction fully
 --echo #
 
+--source include/have_log_bin.inc
 --source include/have_innodb.inc
 --source include/count_sessions.inc
 
@@ -45,8 +46,55 @@ UPDATE t1 SET col2=100 where col1=2;
 SELECT @@in_transaction;
 ROLLBACK;
 
---disconnect con1
+SELECT * FROM t1;
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-37141 DML committed within XA transaction block after deadlock error and implicit rollback
+--echo #
+
+CREATE TABLE t1(col1 INT PRIMARY KEY, col2 INT) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 1), (2, 2);
+SELECT * FROM t1;
+
+--connection con1
+XA BEGIN 'x1';
+--echo # XA Trx-1: Lock 1st record
+UPDATE t1 SET col2=10 where col1=1;
+
+--connection default
+XA BEGIN 'x2';
+--echo # XA Trx-2: Lock 2nd record
+UPDATE t1 SET col2=100 where col1=2;
+
+--connection con1
+--echo # XA Trx-1: Try locking 1st record : Wait
+--send UPDATE t1 SET col2=10 where col1=2
 
+--connection default
+--echo # Wait for XA Trx-1 to get into lock wait stage
+let $wait_condition=
+  SELECT COUNT(*) >= 2 FROM INFORMATION_SCHEMA.INNODB_LOCKS
+  WHERE lock_table like "%t1%";
+--source include/wait_condition.inc
+
+--echo # XA Trx-2: Try locking 2nd record : Deadlock
+--error ER_LOCK_DEADLOCK
+UPDATE t1 SET col2=100 where col1=1;
+--error ER_XAER_RMFAIL
+INSERT INTO t1 VALUES (3, 3), (4, 4);
+--error ER_XAER_RMFAIL
+XA END 'x2';
+XA ROLLBACK 'x2';
+
+--connection con1
+--reap
+XA END 'x1';
+XA ROLLBACK 'x1';
+
+--connection default
 SELECT * FROM t1;
 DROP TABLE t1;
+
+--disconnect con1
 --source include/wait_until_count_sessions.inc

sql/sql_parse.cc

@@ -6248,6 +6248,9 @@ mysql_execute_command(THD *thd, bool is_called_from_prepared_stmt)
       one of storage engines (e.g. due to deadlock). Rollback transaction in
       all storage engines including binary log.
     */
+    auto &xid_state= thd->transaction->xid_state;
+    if (xid_state.is_explicit_XA())
+      xid_state.set_rollback_only();
     trans_rollback_implicit(thd);
     thd->release_transactional_locks();
   }