MDEV-28727 ALTER TABLE ALGORITHM=NOCOPY does not work after upgrade

MDEV-20704 changed the rules of how (HA_BINARY_PACK_KEY |
HA_VAR_LENGTH_KEY) flags are added. Older FRMs before that fix had
these flags for DOUBLE index. After that fix when ALTER sees such old
FRM it thinks it cannot do instant alter because of failed
compare_keys_but_name(): it compares flags against tmp table created
by ALTER.

MDEV-20704 fix was actually not about DOUBLE type but about
FIELDFLAG_BLOB which affected DOUBLE. So there is no direct knowledge
that any other types were not affected.

The proposed fix under CHECK TABLE checks if FRM has
(HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY) flags and was created prior
MDEV-20704 and if so issues "needs upgrade". When mysqlcheck and
mysql_upgrade see such status they issue ALTER TABLE FORCE and upgrade
the table to the version of server.

mysql-test/main/mysql_upgrade.result

@@ -947,4 +947,42 @@ disconnect con1;
 connection default;
 drop table mysql.global_priv;
 rename table mysql.global_priv_bak to mysql.global_priv;
+#
+# MDEV-28727 ALTER TABLE ALGORITHM=NOCOPY does not work after upgrade
+#
+create or replace table pet4 (
+build_time double(18, 7) default null,
+key idx1 (build_time)
+) engine innodb;
+check table pet4;
+Table	Op	Msg_type	Msg_text
+test.pet4	check	error	Table rebuild required. Please do "ALTER TABLE `pet4` FORCE" or dump/reload to fix it!
+check table pet4 for upgrade;
+Table	Op	Msg_type	Msg_text
+test.pet4	check	error	Table rebuild required. Please do "ALTER TABLE `pet4` FORCE" or dump/reload to fix it!
+alter table pet4 add i1 int, algorithm=nocopy;
+ERROR 0A000: ALGORITHM=NOCOPY is not supported for this operation. Try ALGORITHM=INPLACE
+# Running mysqlcheck
+test.pet4
+error    : Table rebuild required. Please do "ALTER TABLE `pet4` FORCE" or dump/reload to fix it!
+
+Repairing tables
+check table pet4;
+Table	Op	Msg_type	Msg_text
+test.pet4	check	status	OK
+alter table pet4 add i1 int, algorithm=nocopy;
+create or replace table pet4 (
+build_time double(18, 7) default null,
+key idx1 (build_time)
+) engine innodb;
+alter table pet4 add i1 int, algorithm=nocopy;
+ERROR 0A000: ALGORITHM=NOCOPY is not supported for this operation. Try ALGORITHM=INPLACE
+# Running mysql_upgrade
+test.pet4
+error    : Table rebuild required. Please do "ALTER TABLE `pet4` FORCE" or dump/reload to fix it!
+check table pet4;
+Table	Op	Msg_type	Msg_text
+test.pet4	check	status	OK
+alter table pet4 add i1 int, algorithm=nocopy;
+drop table pet4;
 # End of 10.4 tests

mysql-test/main/mysql_upgrade.test

@@ -471,4 +471,45 @@ drop table mysql.global_priv;
 rename table mysql.global_priv_bak to mysql.global_priv;
 --remove_file $MYSQLD_DATADIR/mysql_upgrade_info
 
+--echo #
+--echo # MDEV-28727 ALTER TABLE ALGORITHM=NOCOPY does not work after upgrade
+--echo #
+create or replace table pet4 (
+  build_time double(18, 7) default null,
+  key idx1 (build_time)
+) engine innodb;
+
+--remove_file $MYSQLD_DATADIR/test/pet4.frm
+--copy_file std_data/mdev-28727-pet4.frm $MYSQLD_DATADIR/test/pet4.frm
+
+check table pet4;
+check table pet4 for upgrade;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+alter table pet4 add i1 int, algorithm=nocopy;
+
+--echo # Running mysqlcheck
+--exec $MYSQL_CHECK --auto-repair --databases test 2>&1
+check table pet4;
+alter table pet4 add i1 int, algorithm=nocopy;
+
+create or replace table pet4 (
+  build_time double(18, 7) default null,
+  key idx1 (build_time)
+) engine innodb;
+
+--remove_file $MYSQLD_DATADIR/test/pet4.frm
+--copy_file std_data/mdev-28727-pet4.frm $MYSQLD_DATADIR/test/pet4.frm
+
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+alter table pet4 add i1 int, algorithm=nocopy;
+
+--echo # Running mysql_upgrade
+--exec $MYSQL_UPGRADE --silent 2>&1
+file_exists $MYSQLD_DATADIR/mysql_upgrade_info;
+check table pet4;
+alter table pet4 add i1 int, algorithm=nocopy;
+
+--remove_file $MYSQLD_DATADIR/mysql_upgrade_info
+drop table pet4;
+
 --echo # End of 10.4 tests

sql/handler.cc

@@ -4370,17 +4370,32 @@ int handler::ha_check(THD *thd, HA_CHECK_OPT *check_opt)
   DBUG_ASSERT(table_share->tmp_table != NO_TMP_TABLE ||
               m_lock_type != F_UNLCK);
 
-  if ((table->s->mysql_version >= MYSQL_VERSION_ID) &&
+  const ulong v= table->s->mysql_version;
+
+  if ((v >= MYSQL_VERSION_ID) &&
       (check_opt->sql_flags & TT_FOR_UPGRADE))
     return 0;
 
-  if (table->s->mysql_version < MYSQL_VERSION_ID)
+  if (v < MYSQL_VERSION_ID)
   {
     if (unlikely((error= check_old_types())))
       return error;
     error= ha_check_for_upgrade(check_opt);
     if (unlikely(error && (error != HA_ADMIN_NEEDS_CHECK)))
       return error;
+    if (table->s->table_category == TABLE_CATEGORY_USER &&
+        (v < 100142 ||
+         (v >= 100200 && v < 100228) ||
+         (v >= 100300 && v < 100319) ||
+         (v >= 100400 && v < 100409)))
+    {
+      for (const KEY *key= table->key_info,
+           *end= table->key_info + table->s->keys; key < end; key++)
+      {
+        if (key->flags & HA_BINARY_PACK_KEY && key->flags & HA_VAR_LENGTH_KEY)
+          return HA_ADMIN_NEEDS_UPGRADE;
+      }
+    }
     if (unlikely(!error && (check_opt->sql_flags & TT_FOR_UPGRADE)))
       return 0;
   }