MDEV-20704 An index on a double column erroneously uses prefix compression

Author: abarkov

mysql-test/r/myisam.result

@@ -2546,3 +2546,26 @@ myisam_block_size	1024
 select @@global.myisam_block_size;
 @@global.myisam_block_size
 1024
+#
+# MDEV-20704 An index on a double column erroneously uses prefix compression
+#
+CREATE TABLE t1 (
+id INT NOT NULL PRIMARY KEY,
+d  DOUBLE,
+KEY (d)
+) ENGINE=MyISAM;
+
+MyISAM file:         MYSQLD_DATADIR/test/t1
+Record format:       Fixed length
+Character set:       latin1_swedish_ci (8)
+Data records:                    0  Deleted blocks:                 0
+Recordlength:                   13
+
+table description:
+Key Start Len Index   Type
+1   2     4   unique  long                 
+2   6     8   multip. double NULL          
+DROP TABLE t1;
+#
+# End of 5.5 tests
+#

mysql-test/t/myisam.test

@@ -1765,3 +1765,22 @@ DROP TABLE t1;
 #
 show variables like 'myisam_block_size';
 select @@global.myisam_block_size;
+
+
+--echo #
+--echo # MDEV-20704 An index on a double column erroneously uses prefix compression
+--echo #
+
+CREATE TABLE t1 (
+  id INT NOT NULL PRIMARY KEY,
+  d  DOUBLE,
+  KEY (d)
+) ENGINE=MyISAM;
+let $MYSQLD_DATADIR= `select @@datadir`;
+--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR
+--exec $MYISAMCHK -d $MYSQLD_DATADIR/test/t1
+DROP TABLE t1;
+
+--echo #
+--echo # End of 5.5 tests
+--echo #

sql/sql_table.cc

@@ -3725,16 +3725,16 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
       /* Use packed keys for long strings on the first column */
       if (!((*db_options) & HA_OPTION_NO_PACK_KEYS) &&
           !((create_info->table_options & HA_OPTION_NO_PACK_KEYS)) &&
-	  (key_part_length >= KEY_DEFAULT_PACK_LENGTH &&
-	   (sql_field->sql_type == MYSQL_TYPE_STRING ||
-	    sql_field->sql_type == MYSQL_TYPE_VARCHAR ||
-	    sql_field->pack_flag & FIELDFLAG_BLOB)))
+          (key_part_length >= KEY_DEFAULT_PACK_LENGTH &&
+           (sql_field->sql_type == MYSQL_TYPE_STRING ||
+            sql_field->sql_type == MYSQL_TYPE_VARCHAR ||
+            f_is_blob(sql_field->pack_flag))))
       {
-	if ((column_nr == 0 && (sql_field->pack_flag & FIELDFLAG_BLOB)) ||
+        if ((column_nr == 0 && f_is_blob(sql_field->pack_flag)) ||
             sql_field->sql_type == MYSQL_TYPE_VARCHAR)
-	  key_info->flags|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY;
-	else
-	  key_info->flags|= HA_PACK_KEY;
+          key_info->flags|= HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY;
+        else
+          key_info->flags|= HA_PACK_KEY;
       }
       /* Check if the key segment is partial, set the key flag accordingly */
       if (key_part_length != sql_field->key_length)