MDEV-18809 Server crash in fields_in_hash_keyinfo or Assertion `key_i…

…nfo->key_part->field->flags & (1<< 30)' failed in setup_keyinfo_hash

Move calling setup_keyinfo_hash until all continue is exhausted.
And also call re_setup_keyinfo_hash for goto err.

mysql-test/main/long_unique_bugs.result

@@ -143,3 +143,18 @@ a
 3
 4
 drop table t1;
+CREATE TABLE t1 (f VARCHAR(4096), UNIQUE(f)) ENGINE=InnoDB;
+ALTER TABLE t1 DROP KEY f, ADD INDEX idx1(f), ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: ADD INDEX. Try ALGORITHM=NOCOPY
+ALTER TABLE t1 ADD KEY idx2(f);
+Warnings:
+Warning	1071	Specified key was too long; max key length is 3072 bytes
+DROP TABLE t1;
+CREATE TABLE t1(a blob , b blob , unique(a,b));
+alter table t1 drop column b;
+ERROR 42000: Key column 'b' doesn't exist in table
+insert into t1 values(1,1);
+insert into t1 values(1,1);
+ERROR 23000: Duplicate entry '1-1' for key 'a'
+alter table t1 add column c int;
+drop table t1;

mysql-test/main/long_unique_bugs.test

@@ -160,3 +160,21 @@ ALTER TABLE t1 ADD UNIQUE INDEX (a);
 SELECT * FROM t1;
 SELECT a FROM t1;
 drop table t1;
+
+#
+# MDEV-18809 Server crash in fields_in_hash_keyinfo or Assertion `key_info->key_part->field->flags
+# & (1<< 30)' failed in setup_keyinfo_hash
+#
+CREATE TABLE t1 (f VARCHAR(4096), UNIQUE(f)) ENGINE=InnoDB;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 DROP KEY f, ADD INDEX idx1(f), ALGORITHM=INSTANT;
+ALTER TABLE t1 ADD KEY idx2(f);
+DROP TABLE t1;
+CREATE TABLE t1(a blob , b blob , unique(a,b));
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+alter table t1 drop column b;
+insert into t1 values(1,1);
+--error ER_DUP_ENTRY
+insert into t1 values(1,1);
+alter table t1 add column c int;
+drop table t1;

sql/sql_table.cc

@@ -8303,11 +8303,6 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
     bool long_hash_key= false;
     if (key_info->flags & HA_INVISIBLE_KEY)
       continue;
-    if (key_info->algorithm == HA_KEY_ALG_LONG_HASH)
-    {
-      setup_keyinfo_hash(key_info);
-      long_hash_key= true;
-    }
     const char *key_name= key_info->name.str;
     Alter_drop *drop;
     drop_it.rewind();
@@ -8338,6 +8333,11 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
       continue;
     }
 
+    if (key_info->algorithm == HA_KEY_ALG_LONG_HASH)
+    {
+      setup_keyinfo_hash(key_info);
+      long_hash_key= true;
+    }
     const char *dropped_key_part= NULL;
     KEY_PART_INFO *key_part= key_info->key_part;
     key_parts.empty();
@@ -8462,6 +8462,11 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
         if (dropped_key_part)
         {
           my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), dropped_key_part);
+          if (long_hash_key)
+          {
+            key_info->algorithm= HA_KEY_ALG_LONG_HASH;
+            re_setup_keyinfo_hash(key_info);
+          }
           goto err;
         }
       }