Enable basic XA between MyRocks and the binlog

This allows basic master crash-safety
- Un-comment and update relevant parts of the code
- Make rocksdb_rpl suite work like other MyRocks testsuites
  (load the MyRocks plugin, don't start if it is not compiled in, etc)
- For now, disable all tests in the rocksdb_rpl suite.
- MariaDB-fication of rpl_rocksdb_2p_crash_recover test.

Author: spetrunia

storage/rocksdb/ha_rocksdb.cc

@@ -170,7 +170,6 @@ static std::shared_ptr<Rdb_tbl_prop_coll_factory> properties_collector_factory;
 Rdb_dict_manager dict_manager;
 Rdb_cf_manager cf_manager;
 Rdb_ddl_manager ddl_manager;
-const char *m_mysql_gtid;
 Rdb_binlog_manager binlog_manager;
 
 #ifndef _WIN32 
@@ -1668,8 +1667,11 @@ class Rdb_transaction {
 public:
   const char *m_mysql_log_file_name;
   my_off_t m_mysql_log_offset;
+#ifdef MARIAROCKS_NOT_YET
+  // TODO: MariaDB probably doesn't need these at all:
   const char *m_mysql_gtid;
   const char *m_mysql_max_gtid;
+#endif
   String m_detailed_error;
   int64_t m_snapshot_timestamp = 0;
   bool m_ddl_transaction;
@@ -1856,13 +1858,10 @@ class Rdb_transaction {
       rollback();
       return true;
     } else {
-#ifdef MARIAROCKS_NOT_YET
-      my_core::thd_binlog_pos(m_thd, &m_mysql_log_file_name,
-                              &m_mysql_log_offset, &m_mysql_gtid,
-                              &m_mysql_max_gtid);
+      mysql_bin_log_commit_pos(m_thd, &m_mysql_log_offset,
+                               &m_mysql_log_file_name);
       binlog_manager.update(m_mysql_log_file_name, m_mysql_log_offset,
-                            m_mysql_max_gtid, get_write_batch());
-#endif
+                            get_write_batch());
       return commit_no_binlog();
     }
   }
@@ -2683,26 +2682,25 @@ static bool rocksdb_flush_wal(handlerton* hton __attribute__((__unused__)))
 */
 static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx)
 {
-#ifdef MARIAROCKS_NOT_YET
 // This is "ASYNC_COMMIT" feature which is only in webscalesql
   bool async=false;
-#endif
 
   Rdb_transaction *&tx = get_tx_from_thd(thd);
   if (!tx->can_prepare()) {
     return HA_EXIT_FAILURE;
   }
-#ifdef MARIAROCKS_NOT_YET // disable prepare/commit
   if (prepare_tx ||
       (!my_core::thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) {
     /* We were instructed to prepare the whole transaction, or
     this is an SQL statement end and autocommit is on */
+#ifdef MARIAROCKS_NOT_YET // disable prepare/commit
     std::vector<st_slave_gtid_info> slave_gtid_info;
     my_core::thd_slave_gtid_info(thd, &slave_gtid_info);
     for (const auto &it : slave_gtid_info) {
       rocksdb::WriteBatchBase *const write_batch = tx->get_blind_write_batch();
       binlog_manager.update_slave_gtid_info(it.id, it.db, it.gtid, write_batch);
     }
+#endif
 
     if (tx->is_two_phase()) {
       if (thd->durability_property == HA_IGNORE_DURABILITY || async) {
@@ -2713,10 +2711,10 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx)
       if (!tx->prepare(rdb_xid_to_string(xid))) {
         return HA_EXIT_FAILURE;
       }
-      if (thd->durability_property == HA_IGNORE_DURABILITY 
+      if (thd->durability_property == HA_IGNORE_DURABILITY )
 #ifdef MARIAROCKS_NOT_YET
           &&
-          THDVAR(thd, flush_log_at_trx_commit)) {
+          THDVAR(thd, flush_log_at_trx_commit)) 
 #endif          
       {
 #ifdef MARIAROCKS_NOT_YET
@@ -2733,7 +2731,6 @@ static int rocksdb_prepare(handlerton* hton, THD* thd, bool prepare_tx)
 
     DEBUG_SYNC(thd, "rocksdb.prepared");
   }
-#endif
   return HA_EXIT_SUCCESS;
 }
 

storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.inc

@@ -0,0 +1,10 @@
+if (`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'rocksdb' AND support IN ('YES', 'DEFAULT', 'ENABLED')`)
+{
+  --skip Test requires engine RocksDB.
+}
+
+--disable_query_log
+# Table statistics can vary depending on when the memtables are flushed, so
+# flush them at the beginning of the test to ensure the test runs consistently.
+set global rocksdb_force_flush_memtable_now = true;
+--enable_query_log

storage/rocksdb/mysql-test/rocksdb_rpl/include/have_rocksdb.opt

@@ -0,0 +1,12 @@
+--loose-enable-rocksdb
+--loose-enable-rocksdb_global_info
+--loose-enable-rocksdb_ddl
+--loose-enable-rocksdb_cf_options
+--loose-enable_rocksdb_perf_context
+--loose-enable_rocksdb_perf_context_global
+--loose-enable-rocksdb_index_file_map
+--loose-enable-rocksdb_dbstats
+--loose-enable-rocksdb_cfstats
+--loose-enable-rocksdb_lock_info
+--loose-enable-rocksdb_trx
+--loose-enable-rocksdb_locks

storage/rocksdb/mysql-test/rocksdb_rpl/my.cnf

@@ -0,0 +1,17 @@
+!include include/default_my.cnf
+
+[server]
+skip-innodb
+default-storage-engine=rocksdb
+
+
+sql-mode=NO_ENGINE_SUBSTITUTION
+explicit-defaults-for-timestamp=1
+loose-rocksdb_lock_wait_timeout=1
+loose-rocksdb_strict_collation_check=0
+
+loose-rocksdb-flush-log-at-trx-commit=0
+
+# The following is to get rid of the harmless 
+# "Deadlock found when trying to get lock" errors, see MDEV-12285.
+log-warnings=1

storage/rocksdb/mysql-test/rocksdb_rpl/r/rpl_rocksdb_2pc_crash_recover.result

@@ -1,19 +1,19 @@
 DROP TABLE IF EXISTS t1;
 create table t1 (a int primary key, msg varchar(255)) engine=rocksdb;
 SET GLOBAL ROCKSDB_ENABLE_2PC = ON;
-SET SESSION debug="d,crash_commit_after_prepare";
+SET SESSION debug_dbug="d,crash_commit_after_prepare";
 insert into t1 values (1, 'dogz');
 select * from t1;
 a	msg
 SET GLOBAL ROCKSDB_ENABLE_2PC = ON;
-SET SESSION debug="d,crash_commit_after_log";
+SET SESSION debug_dbug="d,crash_commit_after_log";
 insert into t1 values (2, 'catz'), (3, 'men');
 select * from t1;
 a	msg
 2	catz
 3	men
 SET GLOBAL ROCKSDB_ENABLE_2PC = ON;
-SET SESSION debug="d,crash_commit_after";
+SET SESSION debug_dbug="d,crash_commit_after";
 insert into t1 values (4, 'cars'), (5, 'foo');
 select * from t1;
 a	msg
@@ -22,7 +22,7 @@ a	msg
 4	cars
 5	foo
 SET GLOBAL ROCKSDB_ENABLE_2PC = OFF;
-SET SESSION debug="d,crash_commit_after_log";
+SET SESSION debug_dbug="d,crash_commit_after_log";
 insert into t1 values (6, 'shipz'), (7, 'tankz');
 select * from t1;
 a	msg
@@ -31,7 +31,7 @@ a	msg
 4	cars
 5	foo
 SET GLOBAL ROCKSDB_ENABLE_2PC = OFF;
-SET SESSION debug="d,crash_commit_after";
+SET SESSION debug_dbug="d,crash_commit_after";
 insert into t1 values (8, 'space'), (9, 'time');
 select * from t1;
 a	msg

storage/rocksdb/mysql-test/rocksdb_rpl/suite.opt

@@ -0,0 +1,2 @@
+--ignore-db-dirs=.rocksdb --plugin-load=$HA_ROCKSDB_SO --default-storage-engine=rocksdb
+

storage/rocksdb/mysql-test/rocksdb_rpl/suite.pm

@@ -0,0 +1,25 @@
+package My::Suite::Rocksdb;
+
+#
+# Note: ../rocksdb_sys_vars/suite.pm file has a similar
+#  function. If you modify this file, consider modifying that one, too.
+#
+@ISA = qw(My::Suite);
+use My::Find;
+use File::Basename;
+use strict;
+
+sub is_default { not $::opt_embedded_server }
+
+my $sst_dump=
+::mtr_exe_maybe_exists(
+  "$::bindir/storage/rocksdb$::opt_vs_config/sst_dump",
+  "$::path_client_bindir/sst_dump");
+return "RocksDB is not compiled, no sst_dump" unless $sst_dump;
+$ENV{MARIAROCKS_SST_DUMP}="$sst_dump";
+
+# Temporarily disable testing under valgrind, due to MDEV-12439 
+return "RocksDB tests disabled under valgrind" if ($::opt_valgrind);
+
+bless { };
+

storage/rocksdb/mysql-test/rocksdb_rpl/t/disabled.def

@@ -0,0 +1,18 @@
+
+# rpl_rocksdb_2pc_crash_recover 
+
+consistent_snapshot_mixed_engines : Didn't try with MariaDB, yet
+multiclient_2pc : Didn't try with MariaDB, yet
+rpl_crash_safe_wal_corrupt : Didn't try with MariaDB, yet
+rpl_gtid_crash_safe : Didn't try with MariaDB, yet
+rpl_gtid_crash_safe_wal_corrupt : Didn't try with MariaDB, yet
+rpl_gtid_rocksdb_sys_header : Didn't try with MariaDB, yet
+rpl_no_unique_check_on_lag : Didn't try with MariaDB, yet
+rpl_no_unique_check_on_lag_mts : Didn't try with MariaDB, yet
+rpl_rocksdb_snapshot : Didn't try with MariaDB, yet
+rpl_rocksdb_snapshot_without_gtid : Didn't try with MariaDB, yet
+rpl_rocksdb_stress_crash : Didn't try with MariaDB, yet
+rpl_skip_trx_api_binlog_format : Didn't try with MariaDB, yet
+singledelete_idempotent_recovery : Didn't try with MariaDB, yet
+singledelete_idempotent_table : Didn't try with MariaDB, yet
+

storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-master.opt

@@ -1 +1 @@
---gtid_mode=ON --enforce_gtid_consistency --log_bin --log_slave_updates --rocksdb_flush_log_at_trx_commit=1 --rocksdb_write_disable_wal=OFF
+--log_bin --log_slave_updates --rocksdb_flush_log_at_trx_commit=1

storage/rocksdb/mysql-test/rocksdb_rpl/t/rpl_rocksdb_2pc_crash_recover-slave.opt

@@ -1 +1 @@
---gtid_mode=ON --enforce_gtid_consistency --log_bin --log_slave_updates
+--log_bin --log_slave_updates