MDEV-37022 Assertion when adding FK to MyISAM/Aria table with a vector index

adding FK to a table may implicitly add an index, thus sort_keys()
must be called to put the vector index last

Author: vuvova

mysql-test/main/vector2.result

@@ -468,4 +468,20 @@ create table t1 (v vector (1) not null,vector index (v));
 insert into t1 values (0xf0a08080);
 insert into t1 values (0xfa000000);
 drop table t1;
+#
+# MDEV-37022 Assertion when adding FK to MyISAM/Aria table with a vector index
+#
+create table t1 (v vector(1) not null,vector (v));
+alter table t1 add x int references t1(x),add y int constraint c references t1(y);
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `v` vector(1) NOT NULL,
+  `x` int(11) DEFAULT NULL,
+  `y` int(11) DEFAULT NULL,
+  KEY `x` (`x`),
+  KEY `c` (`y`),
+  VECTOR KEY `v` (`v`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_uca1400_ai_ci
+drop table t1;
 # End of 11.8 tests

mysql-test/main/vector2.test

@@ -347,4 +347,12 @@ insert into t1 values (0xf0a08080);
 insert into t1 values (0xfa000000);
 drop table t1;
 
+--echo #
+--echo # MDEV-37022 Assertion when adding FK to MyISAM/Aria table with a vector index
+--echo #
+create table t1 (v vector(1) not null,vector (v));
+alter table t1 add x int references t1(x),add y int constraint c references t1(y);
+show create table t1;
+drop table t1;
+
 --echo # End of 11.8 tests

sql/sql_table.cc

@@ -11471,7 +11471,8 @@ do_continue:;
 
   tmp_disable_binlog(thd);
   create_info->options|=HA_CREATE_TMP_ALTER;
-  if (!(alter_info->flags & ALTER_ADD_INDEX) && !alter_ctx.modified_primary_key)
+  if (!(alter_info->flags & (ALTER_ADD_INDEX|ALTER_ADD_FOREIGN_KEY)) &&
+      !alter_ctx.modified_primary_key)
     create_info->options|= HA_SKIP_KEY_SORT;
   else
     alter_info->flags|= ALTER_INDEX_ORDER;