MDEV-35061 XA PREPARE "not supported by the engine" from storage engine mhnsw, memory leak

disallow explicit XA PREPARE over mhnsw indexes

Author: vuvova

mysql-test/main/vector_innodb.result

@@ -171,3 +171,18 @@ insert into t values (1,x'00000000'),(2,x'00000000');
 lock table t write;
 delete from t;
 drop table t;
+#
+# MDEV-35061 XA PREPARE "not supported by the engine" from storage engine mhnsw, memory leak
+#
+connect con1,localhost,root;
+create table t (a int, v blob not null, vector(v)) engine=innodb;
+xa start 'x';
+insert into t values (1,x'00000000');
+xa end 'x';
+xa prepare 'x';
+ERROR HY000: Got error 138 "Unsupported extension used for table" from storage engine mhnsw
+disconnect con1;
+connection default;
+xa recover;
+formatID	gtrid_length	bqual_length	data
+DROP TABLE t;

mysql-test/main/vector_innodb.test

@@ -159,3 +159,19 @@ insert into t values (1,x'00000000'),(2,x'00000000');
 lock table t write;
 delete from t;
 drop table t;
+
+--echo #
+--echo # MDEV-35061 XA PREPARE "not supported by the engine" from storage engine mhnsw, memory leak
+--echo #
+connect con1,localhost,root;
+create table t (a int, v blob not null, vector(v)) engine=innodb;
+xa start 'x';
+insert into t values (1,x'00000000');
+xa end 'x';
+--error ER_GET_ERRNO
+xa prepare 'x';
+disconnect con1;
+connection default;
+#xa commit 'x';
+xa recover;
+DROP TABLE t;

sql/handler.cc

@@ -1508,9 +1508,7 @@ static int prepare_or_error(transaction_participant *ht, THD *thd, bool all)
   int err= ht->prepare(thd, all);
   status_var_increment(thd->status_var.ha_prepare_count);
   if (err)
-  {
-      my_error(ER_ERROR_DURING_COMMIT, MYF(0), err);
-  }
+      my_error(ER_GET_ERRNO, MYF(0), err, hton_name(ht)->str);
 #ifdef WITH_WSREP
   if (run_wsrep_hooks && !err && ht->flags & HTON_WSREP_REPLICATION &&
       wsrep_after_prepare(thd, all))

sql/log.h

@@ -1476,7 +1476,7 @@ inline bool normalize_binlog_name(char *to, const char *from, bool is_relay_log)
 
 static inline TC_LOG *get_tc_log_implementation()
 {
-  if (total_ha_2pc <= 1)
+  if (total_ha_2pc <= 2) // online_alter_tp and MHNSW_Trx::tp
     return &tc_log_dummy;
   if (opt_bin_log)
     return &mysql_bin_log;

sql/vector_mhnsw.cc

@@ -520,6 +520,7 @@ class MHNSW_Trx : public MHNSW_Share
   static int do_commit(THD *thd, bool);
   static int do_savepoint_rollback(THD *thd, void *);
   static int do_rollback(THD *thd, bool);
+  static int do_prepare(THD *thd, bool);
 };
 
 struct transaction_participant MHNSW_Trx::tp=
@@ -531,7 +532,7 @@ struct transaction_participant MHNSW_Trx::tp=
   [](THD *thd){ return true; },   /*savepoint_rollback_can_release_mdl*/
   nullptr,                        /*savepoint_release*/
   MHNSW_Trx::do_commit, MHNSW_Trx::do_rollback,
-  nullptr,                        /* prepare */
+  MHNSW_Trx::do_prepare,          /* prepare */
   nullptr,                        /* recover */
   nullptr, nullptr,               /* commit/rollback_by_xid */
   nullptr, nullptr,               /* recover_rollback_by_xid/recovery_done */
@@ -606,6 +607,13 @@ int MHNSW_Trx::do_commit(THD *thd, bool)
   return 0;
 }
 
+int MHNSW_Trx::do_prepare(THD *thd, bool)
+{
+  /* Explicit XA is not supported yet */
+  return thd->transaction->xid_state.is_explicit_XA()
+         ? HA_ERR_UNSUPPORTED : 0;
+}
+
 MHNSW_Trx *MHNSW_Trx::get_from_thd(TABLE *table, bool for_update)
 {
   if (!table->file->has_transactions())