MDEV-35219 Unexpected ER_DUP_KEY after OPTIMIZE on MyISAM table with vector key

in-engine optimize can break hlindexes. let's fallback to ALTER

Author: vuvova

mysql-test/main/vector2.result

@@ -292,3 +292,15 @@ d	count(*)
 1	2
 2	1
 drop table t;
+#
+# MDEV-35219 Unexpected ER_DUP_KEY after OPTIMIZE on MyISAM table with vector key
+#
+create table t (v vector(1) not null default 0x30303030, vector(v)) engine=myisam;
+insert into t () values (),(),(),();
+delete from t limit 1;
+optimize table t;
+Table	Op	Msg_type	Msg_text
+test.t	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t	optimize	status	OK
+insert into t select * from t;
+drop table t;

mysql-test/main/vector2.test

@@ -226,3 +226,13 @@ create table t (a int, v vector(1) not null, primary key (a), vector(v));
 insert into t values (1,vec_fromtext('[-1]')),(2,vec_fromtext('[1]')),(3,vec_fromtext('[2]'));
 select vec_distance_euclidean(v,vec_fromtext('[0]')) d, count(*) from t group by d order by d limit 2;
 drop table t;
+
+--echo #
+--echo # MDEV-35219 Unexpected ER_DUP_KEY after OPTIMIZE on MyISAM table with vector key
+--echo #
+create table t (v vector(1) not null default 0x30303030, vector(v)) engine=myisam;
+insert into t () values (),(),(),();
+delete from t limit 1;
+optimize table t;
+insert into t select * from t;
+drop table t;

sql/handler.cc

@@ -5534,7 +5534,8 @@ handler::ha_optimize(THD* thd, HA_CHECK_OPT* check_opt)
               m_lock_type == F_WRLCK);
   mark_trx_read_write();
 
-  return optimize(thd, check_opt);
+  // in-engine optimize can modify rowids, which will break hlindexes
+  return table->s->hlindexes() ? HA_ADMIN_TRY_ALTER : optimize(thd, check_opt);
 }