MDEV-18048: Relax a too strict debug assertion

This assertion should have been relaxed when implementing the first part of
MDEV-15563: instant removal of NOT NULL attribute for ROW_FORMAT=REDUNDANT
tables.

For ROW_FORMAT=REDUNDANT, there is no bitmap of null columns;
the null flags are encoded in the end offset of each field.
We do not really care about the number of fields that can be NULL.

mysql-test/suite/innodb/r/instant_alter.result

@@ -620,6 +620,10 @@ ALTER TABLE t1 DROP PRIMARY KEY;
 ALTER TABLE t1 CHANGE COLUMN a a INT;
 DELETE FROM t1 WHERE a = NULL OR a IS NULL;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
+e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1186,6 +1190,10 @@ ALTER TABLE t1 DROP PRIMARY KEY;
 ALTER TABLE t1 CHANGE COLUMN a a INT;
 DELETE FROM t1 WHERE a = NULL OR a IS NULL;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
+e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1752,10 +1760,14 @@ ALTER TABLE t1 DROP PRIMARY KEY;
 ALTER TABLE t1 CHANGE COLUMN a a INT;
 DELETE FROM t1 WHERE a = NULL OR a IS NULL;
 DROP TABLE t1;
+CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
+e INT, f INT, g INT, h INT, j INT) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
+DROP TABLE t1;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-120
+121
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;

mysql-test/suite/innodb/t/instant_alter.test

@@ -497,6 +497,12 @@ ALTER TABLE t1 CHANGE COLUMN a a INT;
 DELETE FROM t1 WHERE a = NULL OR a IS NULL;
 DROP TABLE t1;
 
+# MDEV-18048 Failing assertion on ALTER
+eval CREATE TABLE t1 (a INT, b INT, c INT NOT NULL, d INT,
+e INT, f INT, g INT, h INT, j INT) $engine;
+ALTER TABLE t1 MODIFY COLUMN c INT, MODIFY COLUMN a INT AFTER b;
+DROP TABLE t1;
+
 dec $format;
 }
 disconnect analyze;

storage/innobase/handler/handler0alter.cc

@@ -390,7 +390,10 @@ inline void dict_table_t::prepare_instant(const dict_table_t& old,
 				goto found_nullable;
 			}
 		}
-		ut_ad(UT_BITS_IN_BYTES(core_null) == oindex.n_core_null_bytes);
+		/* The n_core_null_bytes only matters for
+		ROW_FORMAT=COMPACT and ROW_FORMAT=DYNAMIC tables. */
+		ut_ad(UT_BITS_IN_BYTES(core_null) == oindex.n_core_null_bytes
+		      || !not_redundant());
 		DBUG_ASSERT(i >= oindex.n_core_fields);
 		DBUG_ASSERT(j <= i);
 		DBUG_ASSERT(n_fields - (i - j) == index.n_fields);