Backport MDEV-13890 from 10.2 (InnoDB/XtraDB shutdown failure)

If InnoDB or XtraDB recovered committed transactions at server
startup, but the processing of recovered transactions was
prevented by innodb_read_only or by innodb_force_recovery,
an assertion would fail at shutdown.

This bug was originally reproduced when Mariabackup executed
InnoDB shutdown after preparing (applying redo log into) a backup.

trx_free_prepared(): Allow TRX_STATE_COMMITTED_IN_MEMORY.

trx_undo_free_prepared(): Allow any undo log state. For transactions
that were resurrected in TRX_STATE_COMMITTED_IN_MEMORY
the undo log state would have been reset by trx_undo_set_state_at_finish().

Author: dr-m

storage/innobase/trx/trx0trx.cc

@@ -309,8 +309,9 @@ trx_free_prepared(
 	trx_t*	trx)	/*!< in, own: trx object */
 {
 	ut_a(trx_state_eq(trx, TRX_STATE_PREPARED)
-	     || (trx_state_eq(trx, TRX_STATE_ACTIVE)
-		 && trx->is_recovered
+	     || (trx->is_recovered
+		 && (trx_state_eq(trx, TRX_STATE_ACTIVE)
+		     || trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY))
 		 && (srv_read_only_mode
 		     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO)));
 	ut_a(trx->magic_n == TRX_MAGIC_N);

storage/innobase/trx/trx0undo.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved.
+Copyright (c) 2014, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2015,6 +2015,12 @@ trx_undo_free_prepared(
 		switch (trx->update_undo->state) {
 		case TRX_UNDO_PREPARED:
 			break;
+		case TRX_UNDO_CACHED:
+		case TRX_UNDO_TO_FREE:
+		case TRX_UNDO_TO_PURGE:
+			ut_ad(trx_state_eq(trx,
+					   TRX_STATE_COMMITTED_IN_MEMORY));
+			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
 			trx->is_recovered=false */
@@ -2033,6 +2039,12 @@ trx_undo_free_prepared(
 		switch (trx->insert_undo->state) {
 		case TRX_UNDO_PREPARED:
 			break;
+		case TRX_UNDO_CACHED:
+		case TRX_UNDO_TO_FREE:
+		case TRX_UNDO_TO_PURGE:
+			ut_ad(trx_state_eq(trx,
+					   TRX_STATE_COMMITTED_IN_MEMORY));
+			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
 			trx->is_recovered=false */

storage/xtradb/trx/trx0trx.cc

@@ -475,8 +475,9 @@ trx_free_prepared(
 	trx_t*	trx)	/*!< in, own: trx object */
 {
 	ut_a(trx_state_eq(trx, TRX_STATE_PREPARED)
-	     || (trx_state_eq(trx, TRX_STATE_ACTIVE)
-		 && trx->is_recovered
+	     || (trx->is_recovered
+		 && (trx_state_eq(trx, TRX_STATE_ACTIVE)
+		     || trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY))
 		 && (srv_read_only_mode
 		     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO)));
 	ut_a(trx->magic_n == TRX_MAGIC_N);

storage/xtradb/trx/trx0undo.cc

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved.
+Copyright (c) 2014, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2015,6 +2015,12 @@ trx_undo_free_prepared(
 		switch (trx->update_undo->state) {
 		case TRX_UNDO_PREPARED:
 			break;
+		case TRX_UNDO_CACHED:
+		case TRX_UNDO_TO_FREE:
+		case TRX_UNDO_TO_PURGE:
+			ut_ad(trx_state_eq(trx,
+					   TRX_STATE_COMMITTED_IN_MEMORY));
+			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
 			trx->is_recovered=false */
@@ -2033,6 +2039,12 @@ trx_undo_free_prepared(
 		switch (trx->insert_undo->state) {
 		case TRX_UNDO_PREPARED:
 			break;
+		case TRX_UNDO_CACHED:
+		case TRX_UNDO_TO_FREE:
+		case TRX_UNDO_TO_PURGE:
+			ut_ad(trx_state_eq(trx,
+					   TRX_STATE_COMMITTED_IN_MEMORY));
+			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
 			trx->is_recovered=false */