WL#11300 Crash-safe XA + binary log

# Executive Summary

The goal of this worklog is to improve the interaction between externally
coordinated transactions (XA) and the server internal transaction
coordination w.r.t order of execution, logging and recovery in both single
and multi-server topologies. More specifically:

1. Propagate the XA transaction state to the topology at the expected
   moment so that nodes don't diverge.
2. Keep the XA transaction state consistent throughout the topology while
   performing work in the given transaction.
3. Keep the XA transaction state consistent throughout the topology after
   any server in the topology crashes, recovers and re-joins the topology.

# User stories

- As a MySQL user and while on a multi-server topology (classical
  replication or GR), I wish for externally coordinated transaction's state
  to be propagated at the proper moment throughout the topology, making
  every server state consistent with each other.
- As a MySQL user and while on a single-server or multi-server topology
  (classical replication or GR), I wish to safely use externally
  coordinated transactions (XA) and be able to recover any server in the
  topology to a consistent state in case of a crash.

Reviewed-By: Harini T. S. <harini.t.s@oracle.com>
Reviewed-By: Pedro Gomes <pedro.gomes@oracle.com>
Reviewed-By: Pedro Ribeiro <pedro.a.ribeiro@oracle.com>
Reviewed-By: Sven Sandberg <sven.sandberg@oracle.com>

Author: Pedro Figueiredo

libbinlogevents/include/control_events.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2021, Oracle and/or its affiliates.
+/* Copyright (c) 2014, 2022, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -583,6 +583,7 @@ class XA_prepare_event : public Binary_log_event {
   */
   static const int MY_XIDDATASIZE = 128;
 
+ public:
   struct MY_XID {
     long formatID;
     long gtrid_length;
@@ -624,6 +625,19 @@ class XA_prepare_event : public Binary_log_event {
   void print_event_info(std::ostream &) override {}
   void print_long_info(std::ostream &) override {}
 #endif
+  /**
+    Whether or not this `XA_prepare_event` represents an `XA COMMIT ONE
+    PHASE`.
+
+    @return true if it's an `XA COMMIT ONE PHASE`, false otherwise.
+   */
+  bool is_one_phase() const;
+  /**
+    Retrieves the content of `my_xid` member variable.
+
+    @return The const-reference to the `my_xid` member variable.
+   */
+  MY_XID const &get_xid() const;
 };
 
 /**

libbinlogevents/src/control_events.cpp

@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2021, Oracle and/or its affiliates.
+/* Copyright (c) 2014, 2022, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -348,6 +348,12 @@ XA_prepare_event::XA_prepare_event(const char *buf,
   BAPI_VOID_RETURN;
 }
 
+bool XA_prepare_event::is_one_phase() const { return this->one_phase; }
+
+XA_prepare_event::MY_XID const &XA_prepare_event::get_xid() const {
+  return this->my_xid;
+}
+
 Transaction_payload_event::Transaction_payload_event(const char *payload,
                                                      uint64_t payload_size,
                                                      uint16_t compression_type,

mysql-test/extra/binlog_tests/recovery/end_scenario.inc

@@ -0,0 +1,52 @@
+# ==== Purpose ====
+#
+# Used by test `binlog_gtid.binlog_gtid_recovery_errors`, it cleans up
+# after a binlog recovery error scenario, by:
+#   1. Reinstating the correct binary log file.
+#   2. Starting the crashed server.
+#   3. Cleaning up server context.
+#
+# ==== Parameters ====
+#
+#   $recovery_scenario
+#     The recovery scenario being executed.
+#
+#   $end_scenario_stmt
+#     The statement to execute after restarting the server and before
+#     cleaning the sever context.
+#
+#   $target_binlog_file
+#     The active binlog file name, as instantiated in
+#     `extra/binlog_tests/recovery/kill_copy_malformed_restart.inc`.
+#
+#   $target_binlog_file_bak
+#     Backup of the binlog file before overwriting with a corrupted one, as
+#     instantiated in
+#     `extra/binlog_tests/recovery/kill_copy_malformed_restart.inc`.
+#
+# ==== References ====
+#
+# WL#11300: Crash-safe XA + binary log
+#
+# Related tests;
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+if ($recovery_scenario == '') {
+  --die ERROR: 'recovery_scenario' parameter needed by 'setup_scenario.inc'
+}
+
+--echo extra/binlog_tests/recovery/end_scenario.inc [$recovery_scenario]
+
+--move_file $target_binlog_file_bak $target_binlog_file
+
+--let $expect_crash_on_start =
+--let $restart_parameters =
+--source include/start_mysqld.inc
+
+if ($end_scenario_stmt != '') {
+  --eval $end_scenario_stmt
+  --let $end_scenario_stmt =
+}
+
+DELETE FROM t1;
+RESET MASTER;

mysql-test/extra/binlog_tests/recovery/kill_copy_malformed_restart.inc

@@ -0,0 +1,43 @@
+# ==== Purpose ====
+#
+# Used by test `binlog_gtid.binlog_gtid_recovery_errors`, it sets up the
+# scenario for a malformed binary log file to be read during recovery, by:
+#   1. Getting the current binary log file name.
+#   2. Crashing the server.
+#   3. Replacing the current binlog file with a malformed binary log file,
+#      tampered to trigger the given error scenario,
+#   4. Restarting the server.
+#
+# ==== Parameters ====
+#
+#   $recovery_scenario
+#     The recovery scenario being executed.
+#
+# ==== References ====
+#
+# WL#11300: Crash-safe XA + binary log
+#
+# Related tests;
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+if ($recovery_scenario == '') {
+  --die ERROR: 'recovery_scenario' parameter needed by 'setup_scenario.inc'
+}
+
+--echo extra/binlog_tests/recovery/kill_copy_malformed_restart.inc [$recovery_scenario]
+
+--let $datadir = `SELECT @@GLOBAL.datadir`
+--let $last_binlog_file = query_get_value(SHOW MASTER STATUS, File, 1)
+--let $target_binlog_file = $datadir$last_binlog_file
+--let $target_binlog_file_bak = $target_binlog_file.bak
+
+--source include/kill_mysqld.inc
+
+--let $malformed_binlog_file = $MYSQL_TEST_DIR/std_data/binlog_recovery/$recovery_scenario.000001
+--copy_file $target_binlog_file $target_binlog_file_bak
+--remove_file $target_binlog_file
+--copy_file $malformed_binlog_file $target_binlog_file
+
+--let $restart_parameters = restart: --debug=+d,eval_force_bin_log_recovery
+--let $expect_crash_on_start = 1
+--source include/start_mysqld.inc

mysql-test/extra/xa_crash_safe_tests/assert_recovery_message.inc

@@ -0,0 +1,34 @@
+# ==== Purpose ====
+#
+# Used by scripts testing XA crash recovery, it searches
+# `performance_schema.error_log` table for one occurrence of the given
+# message.
+#
+# ==== Parameters ====
+#
+#   $assert_select
+#     Sub-string to search for in the error log.
+#
+# ==== References ====
+#
+# WL#11300: Crash-safe XA + binary log
+#
+# Related tests;
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+
+if ($assert_select == '') {
+  --die ERROR IN TEST: You must set $assert_select before sourcing assert_recovery_message.inc
+}
+
+let $_row_count_to_assert =
+`SELECT COUNT(1)
+ FROM performance_schema.error_log
+ WHERE
+   (subsystem = 'Server' OR
+    subsystem = 'InnoDB') AND
+   prio = 'Note' AND
+   REGEXP_LIKE(data, '.*$assert_select.*') = 1`;
+--let $assert_text = Found 1 log message(s) for "$assert_select"
+--let $assert_cond = $_row_count_to_assert = 1
+--source include/assert.inc

mysql-test/extra/xa_crash_safe_tests/assert_row_count.inc

@@ -0,0 +1,23 @@
+# ==== Purpose ====
+#
+# Used by scripts testing XA crash recovery, it checks that the table used
+# in those tests has a record count equal to the parameter.
+#
+# ==== Parameters ====
+#
+# $expected_row_count
+#   The expected record count in the table used in XA crash safe tests.
+#
+# For other parameters, see `include/assert.inc`
+#
+# ==== References ====
+#
+#
+# Related tests;
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+
+--let $count = `SELECT COUNT(1) FROM t1;`
+--let $assert_text = Table 'test.t1' record count must be $expected_row_count
+--let $assert_cond = $count = $expected_row_count
+--source include/assert.inc

mysql-test/extra/xa_crash_safe_tests/assert_se_recovery_message.inc

@@ -0,0 +1,33 @@
+# ==== Purpose ====
+#
+# Used by scripts testing XA crash recovery, it searches
+# `performance_schema.error_log` table for one occurrence of the given
+# InnoDB message.
+#
+# ==== Parameters ====
+#
+#   $assert_select
+#     Sub-string to search for in the error log.
+#
+# ==== References ====
+#
+# WL#11300: Crash-safe XA + binary log
+#
+# Related tests;
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+
+if ($assert_select == '') {
+  --die ERROR IN TEST: You must set $assert_select before sourcing assert_recovery_message.inc
+}
+
+let $_row_count_to_assert = 
+`SELECT COUNT(1) 
+ FROM performance_schema.error_log 
+ WHERE 
+   subsystem = 'InnoDB' AND 
+   prio = 'Note' AND 
+   data LIKE '%$assert_select%'`;
+--let $assert_text = Found log message(s) for "$assert_select"
+--let $assert_cond = $_row_count_to_assert > 0
+--source include/assert.inc

mysql-test/extra/xa_crash_safe_tests/assert_xa_recover.inc

@@ -0,0 +1,57 @@
+# ==== Purpose ====
+#
+# Used by scripts testing XA crash recovery, it checks the line count
+# returnes by `XA RECOVER` against the given count parameter and, if
+# different from zero, tries to find the given XID in the list.
+#
+# ==== Parameters ====
+#
+# $expected_prepared_xa_count
+#   The expected line count return by `XA RECOVER`
+#
+# $xid_data
+#   The XID raw data, to find in the `XA RECOVER` returned list. If
+#   `$expected_prepared_xa_count` is non-zero, will try to assert that the
+#   provided XID exists in the `XA RECOVER` outputted list. Support for
+#   more than one XID to test isn't provided.
+#
+# ==== References ====
+#
+# Related tests:
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+
+--let $statement = XA RECOVER
+--let $column = data
+--source include/get_row_count.inc
+
+if ($expected_prepared_xa_count == 0) {
+  --let $assert_text = No XA transactions in PREPARED state
+}
+if ($expected_prepared_xa_count != 0) {
+  --let $assert_text = $expected_prepared_xa_count XA transaction(s) in PREPARED state
+}
+# Test `XA RECOVER` returned line count
+--let $assert_cond = "$row_count" = "$expected_prepared_xa_count"
+--source include/assert.inc
+
+if ($expected_prepared_xa_count != 0) {
+  if ($xid_data != '') {
+    # For each line returned from `XA RECOVER`, try to find the given XID
+    --let $row_number = $row_count
+    while($row_number != 0) {
+      --let $pending_xa = query_get_value(XA RECOVER, data, $row_number)
+      if ($pending_xa != $xid_data) {
+        --let $pending_xa =
+        --dec $row_number
+      }
+      if ($pending_xa == $xid_data) {
+        # If found, exit loop
+        --let $row_number = 0
+      }
+    }
+    --let $assert_text = XA transaction $xid is in PREPARED state
+    --let $assert_cond = "$pending_xa" = "$xid_data"
+    --source include/assert.inc
+  }
+}

mysql-test/extra/xa_crash_safe_tests/end_utility_functions.inc

@@ -0,0 +1,16 @@
+# ==== Purpose ====
+#
+# Used by scripts testing XA crash recovery, it cleans up temporary files
+# created by `extra/xa_crash_safe_tests/utility_functions.inc`.
+#
+# ==== References ====
+#
+# WL#11300: Crash-safe XA + binary log
+#
+# Related tests;
+#   see extra/xa_crash_safe_tests/setup.inc
+#
+--remove_file $exec_xa_transaction_body
+--remove_file $clean_up_xa_transactions
+--remove_file $error_on_recovery_for_normal_transaction
+--remove_file $error_on_recovery_for_xa_transaction