MDEV-18623 Assertion after DROP FULLTEXT INDEX and removing NOT NULL

instant_alter_column_possible(): Do not support instantaneous removal
of NOT NULL if the table needs to be rebuilt for removing the hidden
FTS_DOC_ID column. This is not ideal and should ultimately be fixed
properly in MDEV-17459.

mysql-test/suite/innodb/r/instant_alter_bugs.result

@@ -193,3 +193,13 @@ INSERT INTO t1 SELECT * FROM t1;
 INSERT INTO t1 SELECT * FROM t1;
 ALTER TABLE t1 ADD vb INT AS (b) VIRTUAL;
 DROP TABLE t1;
+#
+# MDEV-18623 Assertion after DROP FULLTEXT INDEX and removing NOT NULL
+#
+CREATE TABLE t1 (c TEXT NOT NULL, FULLTEXT INDEX ftidx(c)) ENGINE=InnoDB
+ROW_FORMAT=REDUNDANT;
+ALTER TABLE t1 DROP INDEX ftidx;
+ALTER TABLE t1 MODIFY c TEXT NULL, ALGORITHM=INSTANT;
+ERROR 0A000: ALGORITHM=INSTANT is not supported for this operation. Try ALGORITHM=INPLACE
+ALTER TABLE t1 MODIFY c TEXT NULL;
+DROP TABLE t1;

mysql-test/suite/innodb/t/instant_alter_bugs.test

@@ -205,3 +205,14 @@ INSERT INTO t1 SELECT * FROM t1;
 # Exploit MDEV-17468 to force the table definition to be reloaded
 ALTER TABLE t1 ADD vb INT AS (b) VIRTUAL;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-18623 Assertion after DROP FULLTEXT INDEX and removing NOT NULL
+--echo #
+CREATE TABLE t1 (c TEXT NOT NULL, FULLTEXT INDEX ftidx(c)) ENGINE=InnoDB
+ROW_FORMAT=REDUNDANT;
+ALTER TABLE t1 DROP INDEX ftidx;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED
+ALTER TABLE t1 MODIFY c TEXT NULL, ALGORITHM=INSTANT;
+ALTER TABLE t1 MODIFY c TEXT NULL;
+DROP TABLE t1;

storage/innobase/handler/handler0alter.cc

@@ -1560,6 +1560,14 @@ instant_alter_column_possible(
 
 	if (ha_alter_info->handler_flags & ALTER_COLUMN_NULLABLE) {
 		if (ib_table.not_redundant()) {
+			/* Instantaneous removal of NOT NULL is
+			only supported for ROW_FORMAT=REDUNDANT. */
+			return false;
+		}
+		if (ib_table.fts_doc_id_index
+		    && !innobase_fulltext_exist(altered_table)) {
+			/* Removing hidden FTS_DOC_ID_INDEX(FTS_DOC_ID)
+			requires that the table be rebuilt. */
 			return false;
 		}