MDEV-27578 DESC attribute upon spatial index creation prevents ER_DUP_INDEX warning

strip DESC from SPATIAL indexes earlier. Before check_duplicate_key()
not when creating the frm

Author: vuvova

mysql-test/main/alter_table_errors.result

@@ -1,3 +1,6 @@
+#
+# MDEV-16110 ALTER with ALGORITHM=INPLACE breaks temporary table with virtual columns
+#
 create table t (a int, v int as (a)) engine=innodb;
 alter table t change column a b tinyint, algorithm=inplace;
 ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
@@ -28,8 +31,23 @@ t2	CREATE TEMPORARY TABLE `t2` (
   `v` int(11) GENERATED ALWAYS AS (`a`) VIRTUAL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
 drop temporary table t1, t2;
+unlock tables;
+#
+# MDEV-18083 ASAN heap-use-after-free in Field::set_warning_truncated_wrong_value upon inserting into temporary table
+#
 create temporary table t1 (a int);
 alter table t1 add column f text;
 insert into t1 values ('x','foo');
 ERROR 22007: Incorrect integer value: 'x' for column `test`.`t1`.`a` at row 1
 drop temporary table t1;
+# End of 10.2 tests
+#
+# MDEV-27578 DESC attribute upon spatial index creation prevents ER_DUP_INDEX warning
+#
+create table t1 (c point not null);
+alter table t1 add spatial index b1(c desc);
+alter table t1 add spatial index b2(c desc);
+Warnings:
+Note	1831	Duplicate index `b2`. This is deprecated and will be disallowed in a future release
+drop table t1;
+# End of 10.11 tests

mysql-test/main/alter_table_errors.test

@@ -1,8 +1,8 @@
 --source include/have_innodb.inc
 
-#
-# MDEV-16110 ALTER with ALGORITHM=INPLACE breaks temporary table with virtual columns
-#
+--echo #
+--echo # MDEV-16110 ALTER with ALGORITHM=INPLACE breaks temporary table with virtual columns
+--echo #
 create table t (a int, v int as (a)) engine=innodb;
 --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t change column a b tinyint, algorithm=inplace;
@@ -20,12 +20,25 @@ lock table t2 write;
 alter table t2 change column a b int, algorithm=inplace;
 show create table t2;
 drop temporary table t1, t2;
+unlock tables;
 
-#
-# MDEV-18083 ASAN heap-use-after-free in Field::set_warning_truncated_wrong_value upon inserting into temporary table
-#
+--echo #
+--echo # MDEV-18083 ASAN heap-use-after-free in Field::set_warning_truncated_wrong_value upon inserting into temporary table
+--echo #
 create temporary table t1 (a int);
 alter table t1 add column f text;
 --error ER_TRUNCATED_WRONG_VALUE_FOR_FIELD
 insert into t1 values ('x','foo');
 drop temporary table t1;
+
+--echo # End of 10.2 tests
+
+--echo #
+--echo # MDEV-27578 DESC attribute upon spatial index creation prevents ER_DUP_INDEX warning
+--echo #
+create table t1 (c point not null);
+alter table t1 add spatial index b1(c desc);
+alter table t1 add spatial index b2(c desc);
+drop table t1;
+
+--echo # End of 10.11 tests

sql/sql_table.cc

@@ -3724,6 +3724,12 @@ mysql_prepare_create_table_finalize(THD *thd, HA_CREATE_INFO *create_info,
         auto_increment_key= sql_field;
       }
 
+      /* For SPATIAL, FULLTEXT and HASH indexes (anything other than B-tree),
+         ignore the ASC/DESC attribute of columns. */
+      if ((key_info->algorithm > HA_KEY_ALG_BTREE) ||
+          (key_info->flags & (HA_SPATIAL|HA_FULLTEXT)))
+        column->asc= true; // ignore DESC
+
       key_part_info->fieldnr= field;
       key_part_info->offset=  (uint16) sql_field->offset;
       key_part_info->key_type=sql_field->pack_flag;

sql/unireg.cc

@@ -685,12 +685,6 @@ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo,
                         key->flags, key->user_defined_key_parts,
                         key->key_part));
 
-    /* For SPATIAL, FULLTEXT and HASH indexes (anything other than B-tree),
-       ignore the ASC/DESC attribute of columns. */
-    const uchar ha_reverse_sort=
-      key->algorithm > HA_KEY_ALG_BTREE || key->flags & (HA_FULLTEXT|HA_SPATIAL)
-      ? 0 : HA_REVERSE_SORT;
-
     for (key_part=key->key_part,key_part_end=key_part+key->user_defined_key_parts ;
 	 key_part != key_part_end ;
 	 key_part++)
@@ -703,14 +697,17 @@ static uint pack_keys(uchar *keybuff, uint key_count, KEY *keyinfo,
       int2store(pos,key_part->fieldnr+1+FIELD_NAME_USED);
       offset= (uint) (key_part->offset+data_offset+1);
       int2store(pos+2, offset);
-      key_part->key_part_flag &= ha_reverse_sort;
+      key_part->key_part_flag &= HA_REVERSE_SORT;
+      /* DESC can be set only for BTREE indexes */
+      DBUG_ASSERT(key_part->key_part_flag == 0 ||
+                  (key->algorithm <= HA_KEY_ALG_BTREE && !(key->flags & (HA_FULLTEXT|HA_SPATIAL))));
       pos[4]= (uchar)(key_part->key_part_flag);
       int2store(pos+5,key_part->key_type);
       int2store(pos+7,key_part->length);
       pos+=9;
     }
   }
-	/* Save keynames */
+  /* Save keynames */
   keyname_pos=pos;
   *pos++=(uchar) NAMES_SEP_CHAR;
   for (key=keyinfo ; key != end ; key++)