MDEV-37068 Can't find record in 't1' on INSERT to Vector table

ER_DUP_ENTRY only rolls back one current statement, it doesn't
abort the transaction. Thus MHNSW_Trx has to register for
"statement transaction" and support "statement rollback".

Author: vuvova

mysql-test/main/vector_innodb.result

@@ -360,4 +360,18 @@ drop table t;
 #
 create table t (c1 varchar(1024) not null,c2 vector(1) not null,  unique key c1(c1), vector key c2 (c2)) engine=innodb default charset=utf8mb3 ;
 ERROR 42000: Primary key was too long for vector indexes, max length is 256 bytes
+#
+# MDEV-37068 Can't find record in 't1' on INSERT to Vector table
+#
+SET sql_mode='';
+create table t1 (v vector (1) not null,vector vec (v),unique vu (v)) engine=innodb;
+start transaction;
+insert t1 values (10), (20);
+ERROR 23000: Duplicate entry '\x00\x00\x00\x00' for key 'vu'
+insert t1 values (1);
+Warnings:
+Warning	4078	Cannot cast 'int' as 'vector' in assignment of `test`.`t1`.`v`
+Warning	1292	Incorrect vector value: '1' for column `test`.`t1`.`v` at row 1
+drop table t1;
+set sql_mode=default;
 # End of 11.8 tests

mysql-test/main/vector_innodb.test

@@ -356,4 +356,16 @@ drop table t;
 --error ER_TOO_LONG_KEY
 create table t (c1 varchar(1024) not null,c2 vector(1) not null,  unique key c1(c1), vector key c2 (c2)) engine=innodb default charset=utf8mb3 ;
 
+--echo #
+--echo # MDEV-37068 Can't find record in 't1' on INSERT to Vector table
+--echo #
+SET sql_mode='';
+create table t1 (v vector (1) not null,vector vec (v),unique vu (v)) engine=innodb;
+start transaction;
+--error ER_DUP_ENTRY
+insert t1 values (10), (20);
+insert t1 values (1);
+drop table t1;
+set sql_mode=default;
+
 --echo # End of 11.8 tests

sql/vector_mhnsw.cc

@@ -650,8 +650,11 @@ int MHNSW_Trx::do_savepoint_rollback(THD *thd, void *)
   return 0;
 }
 
-int MHNSW_Trx::do_rollback(THD *thd, bool)
+int MHNSW_Trx::do_rollback(THD *thd, bool all)
 {
+  if (!all && thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))
+    return do_savepoint_rollback(thd, nullptr);
+
   MHNSW_Trx *trx_next;
   for (auto trx= static_cast<MHNSW_Trx*>(thd_get_ha_data(thd, &tp));
        trx; trx= trx_next)
@@ -663,8 +666,11 @@ int MHNSW_Trx::do_rollback(THD *thd, bool)
   return 0;
 }
 
-int MHNSW_Trx::do_commit(THD *thd, bool)
+int MHNSW_Trx::do_commit(THD *thd, bool all)
 {
+  if (!all && thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))
+    return 0;
+
   MHNSW_Trx *trx_next;
   for (auto trx= static_cast<MHNSW_Trx*>(thd_get_ha_data(thd, &tp));
        trx; trx= trx_next)
@@ -734,8 +740,9 @@ MHNSW_Trx *MHNSW_Trx::get_from_thd(TABLE *table, bool for_update)
     thd_set_ha_data(thd, &tp, trx);
     if (!trx->next)
     {
-      bool all= thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN);
-      trans_register_ha(thd, all, &tp, 0);
+      if (thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))
+        trans_register_ha(thd, true, &tp, 0);
+      trans_register_ha(thd, false, &tp, 0);
     }
   }
   trx->refcnt++;