MDEV-18719 Assertion (c.prtype ^ o->prtype) & ... failed on ALTER TABLE

The prtype & DATA_LONG_TRUE_VARCHAR flag only plays a role when
converting between InnoDB internal format and the MariaDB SQL layer
row format. Ideally this flag would never have been persisted in the
InnoDB data dictionary.

There were bogus assertion failures when an instant ADD, DROP, or
column reordering was combined with a change of extending a VARCHAR
from less than 256 bytes to more than 255 bytes. Such changes are
allowed starting with MDEV-15563 in MariaDB 10.4.3.

dict_table_t::instant_column(), dict_col_t::same_format(): Ignore
the DATA_LONG_TRUE_VARCHAR flag, because it does not affect the
persistent storage format.

mysql-test/suite/innodb/r/instant_alter.result

@@ -869,6 +869,15 @@ a	b	c
 2	1	2
 3	2	4
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+a	b	c
+1	a	NULL
+2	bah	3
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -1684,6 +1693,15 @@ a	b	c
 2	1	2
 3	2	4
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) ENGINE=InnoDB ROW_FORMAT=COMPACT;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+a	b	c
+1	a	NULL
+2	bah	3
+DROP TABLE t1;
 CREATE TABLE t1
 (id INT PRIMARY KEY, c2 INT UNIQUE,
 c3 POINT NOT NULL DEFAULT ST_GeomFromText('POINT(3 4)'),
@@ -2499,10 +2517,19 @@ a	b	c
 2	1	2
 3	2	4
 DROP TABLE t1;
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+a	b	c
+1	a	NULL
+2	bah	3
+DROP TABLE t1;
 disconnect analyze;
 SELECT variable_value-@old_instant instants
 FROM information_schema.global_status
 WHERE variable_name = 'innodb_instant_alter_column';
 instants
-178
+181
 SET GLOBAL innodb_purge_rseg_truncate_frequency= @saved_frequency;

mysql-test/suite/innodb/t/instant_alter.test

@@ -737,6 +737,14 @@ UPDATE t1 SET c=c+1;
 SELECT * FROM t1;
 DROP TABLE t1;
 
+# MDEV-18719 Assertion (c.prtype ^ o->prtype) & ... failed on ALTER TABLE
+eval CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(1)) $engine;
+INSERT INTO t1 VALUES(1,'a');
+ALTER TABLE t1 MODIFY b VARCHAR(256), ADD COLUMN c INT;
+INSERT INTO t1 VALUES(2,'bah',3);
+SELECT * FROM t1;
+DROP TABLE t1;
+
 dec $format;
 }
 disconnect analyze;

storage/innobase/handler/handler0alter.cc

@@ -536,7 +536,8 @@ inline bool dict_table_t::instant_column(const dict_table_t& table,
 		if (const dict_col_t* o = find(old_cols, col_map, n_cols, i)) {
 			c.def_val = o->def_val;
 			DBUG_ASSERT(!((c.prtype ^ o->prtype)
-				      & ~(DATA_NOT_NULL | DATA_VERSIONED)));
+				      & ~(DATA_NOT_NULL | DATA_VERSIONED
+					  | DATA_LONG_TRUE_VARCHAR)));
 			DBUG_ASSERT(c.mtype == o->mtype);
 			DBUG_ASSERT(c.len >= o->len);
 

storage/innobase/include/dict0mem.h

@@ -693,7 +693,8 @@ struct dict_col_t{
 			&& mbminlen == other.mbminlen
 			&& mbmaxlen == other.mbmaxlen
 			&& !((prtype ^ other.prtype)
-			     & ~(DATA_NOT_NULL | DATA_VERSIONED));
+			     & ~(DATA_NOT_NULL | DATA_VERSIONED
+				 | DATA_LONG_TRUE_VARCHAR));
 	}
 };