MDEV-15772 Potential list overrun during XA recovery

InnoDB could return the same list again and again if the buffer
passed to trx_recover_for_mysql() is smaller than the number of
transactions that InnoDB recovered in XA PREPARE state.

We introduce the transaction state TRX_PREPARED_RECOVERED, which
is like TRX_PREPARED, but will be set during trx_recover_for_mysql()
so that each transaction will only be returned once.

Because init_server_components() is invoking ha_recover() twice,
we must reset the state of the transactions back to TRX_PREPARED
after returning the complete list, so that repeated traversals
will see the complete list again, instead of seeing an empty list.
Without this tweak, the test main.tc_heuristic_recover would hang
in MariaDB 10.1.

mysql-test/suite/innodb/r/xa_debug.result

@@ -0,0 +1,310 @@
+call mtr.add_suppression("Found 50 prepared XA transactions");
+create table t1 (a int) engine=innodb;
+insert into t1 values(1);
+xa start 'test50';
+insert into t1 values(1);
+xa end 'test50';
+xa prepare 'test50';
+xa start 'test49';
+insert into t1 values(1);
+xa end 'test49';
+xa prepare 'test49';
+xa start 'test48';
+insert into t1 values(1);
+xa end 'test48';
+xa prepare 'test48';
+xa start 'test47';
+insert into t1 values(1);
+xa end 'test47';
+xa prepare 'test47';
+xa start 'test46';
+insert into t1 values(1);
+xa end 'test46';
+xa prepare 'test46';
+xa start 'test45';
+insert into t1 values(1);
+xa end 'test45';
+xa prepare 'test45';
+xa start 'test44';
+insert into t1 values(1);
+xa end 'test44';
+xa prepare 'test44';
+xa start 'test43';
+insert into t1 values(1);
+xa end 'test43';
+xa prepare 'test43';
+xa start 'test42';
+insert into t1 values(1);
+xa end 'test42';
+xa prepare 'test42';
+xa start 'test41';
+insert into t1 values(1);
+xa end 'test41';
+xa prepare 'test41';
+xa start 'test40';
+insert into t1 values(1);
+xa end 'test40';
+xa prepare 'test40';
+xa start 'test39';
+insert into t1 values(1);
+xa end 'test39';
+xa prepare 'test39';
+xa start 'test38';
+insert into t1 values(1);
+xa end 'test38';
+xa prepare 'test38';
+xa start 'test37';
+insert into t1 values(1);
+xa end 'test37';
+xa prepare 'test37';
+xa start 'test36';
+insert into t1 values(1);
+xa end 'test36';
+xa prepare 'test36';
+xa start 'test35';
+insert into t1 values(1);
+xa end 'test35';
+xa prepare 'test35';
+xa start 'test34';
+insert into t1 values(1);
+xa end 'test34';
+xa prepare 'test34';
+xa start 'test33';
+insert into t1 values(1);
+xa end 'test33';
+xa prepare 'test33';
+xa start 'test32';
+insert into t1 values(1);
+xa end 'test32';
+xa prepare 'test32';
+xa start 'test31';
+insert into t1 values(1);
+xa end 'test31';
+xa prepare 'test31';
+xa start 'test30';
+insert into t1 values(1);
+xa end 'test30';
+xa prepare 'test30';
+xa start 'test29';
+insert into t1 values(1);
+xa end 'test29';
+xa prepare 'test29';
+xa start 'test28';
+insert into t1 values(1);
+xa end 'test28';
+xa prepare 'test28';
+xa start 'test27';
+insert into t1 values(1);
+xa end 'test27';
+xa prepare 'test27';
+xa start 'test26';
+insert into t1 values(1);
+xa end 'test26';
+xa prepare 'test26';
+xa start 'test25';
+insert into t1 values(1);
+xa end 'test25';
+xa prepare 'test25';
+xa start 'test24';
+insert into t1 values(1);
+xa end 'test24';
+xa prepare 'test24';
+xa start 'test23';
+insert into t1 values(1);
+xa end 'test23';
+xa prepare 'test23';
+xa start 'test22';
+insert into t1 values(1);
+xa end 'test22';
+xa prepare 'test22';
+xa start 'test21';
+insert into t1 values(1);
+xa end 'test21';
+xa prepare 'test21';
+xa start 'test20';
+insert into t1 values(1);
+xa end 'test20';
+xa prepare 'test20';
+xa start 'test19';
+insert into t1 values(1);
+xa end 'test19';
+xa prepare 'test19';
+xa start 'test18';
+insert into t1 values(1);
+xa end 'test18';
+xa prepare 'test18';
+xa start 'test17';
+insert into t1 values(1);
+xa end 'test17';
+xa prepare 'test17';
+xa start 'test16';
+insert into t1 values(1);
+xa end 'test16';
+xa prepare 'test16';
+xa start 'test15';
+insert into t1 values(1);
+xa end 'test15';
+xa prepare 'test15';
+xa start 'test14';
+insert into t1 values(1);
+xa end 'test14';
+xa prepare 'test14';
+xa start 'test13';
+insert into t1 values(1);
+xa end 'test13';
+xa prepare 'test13';
+xa start 'test12';
+insert into t1 values(1);
+xa end 'test12';
+xa prepare 'test12';
+xa start 'test11';
+insert into t1 values(1);
+xa end 'test11';
+xa prepare 'test11';
+xa start 'test10';
+insert into t1 values(1);
+xa end 'test10';
+xa prepare 'test10';
+xa start 'test9';
+insert into t1 values(1);
+xa end 'test9';
+xa prepare 'test9';
+xa start 'test8';
+insert into t1 values(1);
+xa end 'test8';
+xa prepare 'test8';
+xa start 'test7';
+insert into t1 values(1);
+xa end 'test7';
+xa prepare 'test7';
+xa start 'test6';
+insert into t1 values(1);
+xa end 'test6';
+xa prepare 'test6';
+xa start 'test5';
+insert into t1 values(1);
+xa end 'test5';
+xa prepare 'test5';
+xa start 'test4';
+insert into t1 values(1);
+xa end 'test4';
+xa prepare 'test4';
+xa start 'test3';
+insert into t1 values(1);
+xa end 'test3';
+xa prepare 'test3';
+xa start 'test2';
+insert into t1 values(1);
+xa end 'test2';
+xa prepare 'test2';
+xa start 'test1';
+insert into t1 values(1);
+xa end 'test1';
+xa prepare 'test1';
+xa recover;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+1	5	0	test2
+1	5	0	test3
+1	5	0	test4
+1	5	0	test5
+1	5	0	test6
+1	5	0	test7
+1	5	0	test8
+1	5	0	test9
+1	6	0	test10
+1	6	0	test11
+1	6	0	test12
+1	6	0	test13
+1	6	0	test14
+1	6	0	test15
+1	6	0	test16
+1	6	0	test17
+1	6	0	test18
+1	6	0	test19
+1	6	0	test20
+1	6	0	test21
+1	6	0	test22
+1	6	0	test23
+1	6	0	test24
+1	6	0	test25
+1	6	0	test26
+1	6	0	test27
+1	6	0	test28
+1	6	0	test29
+1	6	0	test30
+1	6	0	test31
+1	6	0	test32
+1	6	0	test33
+1	6	0	test34
+1	6	0	test35
+1	6	0	test36
+1	6	0	test37
+1	6	0	test38
+1	6	0	test39
+1	6	0	test40
+1	6	0	test41
+1	6	0	test42
+1	6	0	test43
+1	6	0	test44
+1	6	0	test45
+1	6	0	test46
+1	6	0	test47
+1	6	0	test48
+1	6	0	test49
+1	6	0	test50
+xa recover;
+formatID	gtrid_length	bqual_length	data
+1	5	0	test1
+1	5	0	test2
+1	5	0	test3
+1	5	0	test4
+1	5	0	test5
+1	5	0	test6
+1	5	0	test7
+1	5	0	test8
+1	5	0	test9
+1	6	0	test10
+1	6	0	test11
+1	6	0	test12
+1	6	0	test13
+1	6	0	test14
+1	6	0	test15
+1	6	0	test16
+1	6	0	test17
+1	6	0	test18
+1	6	0	test19
+1	6	0	test20
+1	6	0	test21
+1	6	0	test22
+1	6	0	test23
+1	6	0	test24
+1	6	0	test25
+1	6	0	test26
+1	6	0	test27
+1	6	0	test28
+1	6	0	test29
+1	6	0	test30
+1	6	0	test31
+1	6	0	test32
+1	6	0	test33
+1	6	0	test34
+1	6	0	test35
+1	6	0	test36
+1	6	0	test37
+1	6	0	test38
+1	6	0	test39
+1	6	0	test40
+1	6	0	test41
+1	6	0	test42
+1	6	0	test43
+1	6	0	test44
+1	6	0	test45
+1	6	0	test46
+1	6	0	test47
+1	6	0	test48
+1	6	0	test49
+1	6	0	test50
+xa recover;
+formatID	gtrid_length	bqual_length	data
+drop table t1;

mysql-test/suite/innodb/t/xa_debug.test

@@ -0,0 +1,45 @@
+-- source include/have_innodb.inc
+-- source include/have_debug.inc
+-- source include/not_embedded.inc
+
+call mtr.add_suppression("Found 50 prepared XA transactions");
+create table t1 (a int) engine=innodb;
+insert into t1 values(1);
+
+let $trial = 50;
+while ($trial)
+{
+--connect (con$trial, localhost, root,,)
+let $st_pre = `select concat('test', $trial)`;
+eval xa start '$st_pre';
+insert into t1 values(1);
+eval xa end '$st_pre';
+eval xa prepare '$st_pre';
+dec $trial;
+}
+
+connection default;
+# Kill and restart the server.
+-- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+-- shutdown_server 0
+-- source include/wait_until_disconnected.inc
+
+-- exec echo "restart:--debug_dbug=+d,min_xa_len" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+-- enable_reconnect
+-- source include/wait_until_connected_again.inc
+-- disable_reconnect
+--sorted_result
+xa recover;
+--sorted_result
+xa recover;
+--disable_query_log
+let $trial = 50;
+while ($trial)
+{
+let $st_pre = `select concat('test', $trial)`;
+eval xa commit '$st_pre';
+dec $trial;
+}
+--enable_query_log
+xa recover;
+drop table t1;

sql/handler.cc

@@ -1740,6 +1740,7 @@ int ha_recover(HASH *commit_list)
   for (info.len= MAX_XID_LIST_SIZE ; 
        info.list==0 && info.len > MIN_XID_LIST_SIZE; info.len/=2)
   {
+    DBUG_EXECUTE_IF("min_xa_len", info.len = 16;);
     info.list=(XID *)my_malloc(info.len*sizeof(XID), MYF(0));
   }
   if (!info.list)

storage/innobase/include/trx0sys.ic

@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2019, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -353,10 +354,13 @@ trx_is_active(
 	}
 
 	trx = trx_get_on_id(trx_id);
-	if (trx && (trx->conc_state == TRX_ACTIVE
-		    || trx->conc_state == TRX_PREPARED)) {
-
-		return(TRUE);
+	if (trx) {
+		switch (trx->conc_state) {
+		case TRX_ACTIVE:
+		case TRX_PREPARED:
+		case TRX_PREPARED_RECOVERED:
+			return(TRUE);
+		}
 	}
 
 	return(FALSE);