MDEV-13795/MDEV-14332 Corruption during online table-rebuilding ALTER…

… when VIRTUAL columns exist

When MySQL 5.7 introduced indexed virtual columns, it introduced
several bugs into the online table-rebuilding ALTER, that is,
the row_log_table_apply() family of functions.

The online_log format that was introduced for online table-rebuilding
ALTER in MySQL 5.6 should be sufficient. Ideally, any indexed virtual
column values would be evaluated based on the log records in the temporary
file. There is no need to log virtual column values.

(For ADD INDEX, that is row_log_apply(), we always must log the values of
the keys, no matter if the columns are virtual.)

Because omitting the virtual column values removes any chance of
row_log_table_apply() working with indexed virtual columns, we
will for now refuse LOCK=NONE in table-rebuilding ALTER operations
when indexes on virtual columns exist. This restriction would be
lifted in MDEV-14341.

innobase_indexed_virtual_exist(): New predicate, to determine if
indexed virtual columns exist in a table definition.

ha_innobase::check_if_supported_inplace_alter(): Refuse online rebuild
if indexed virtual columns exist.

rec_get_converted_size_temp_v(), rec_convert_dtuple_to_temp_v(): Remove.

row_log_table_delete(), row_log_table_update(, row_log_table_insert():
Remove parameters for virtual columns.

trx_undo_read_v_rows(): Remove the col_map parameter.

row_log_table_apply(): Do not deal with virtual columns.

mysql-test/suite/gcol/r/innodb_virtual_basic.result

@@ -960,9 +960,12 @@ INSERT INTO t1 VALUES (11, 3, DEFAULT, 'mm');
 INSERT INTO t1 VALUES (18, 1, DEFAULT, 'mm');
 INSERT INTO t1 VALUES (28, 1, DEFAULT, 'mm');
 ALTER TABLE t1 ADD INDEX idx12 (c) , FORCE, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t1 ADD INDEX idx12 (c), LOCK=NONE;
 ALTER TABLE t1 DROP COLUMN h,  ADD INDEX idx (c) , FORCE, LOCK=NONE;
-Warnings:
-Note	1831	Duplicate index `idx`. This is deprecated and will be disallowed in a future release
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t1 DROP COLUMN h,  ADD INDEX idx (c), LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
 DROP TABLE t1 ;
 CREATE TABLE t1 (a INT, b INT, c INT GENERATED ALWAYS AS(a+b), d INT GENERATED ALWAYS AS(a+b), h VARCHAR(10));
 INSERT INTO t1 VALUES (11, 3, DEFAULT, DEFAULT, 'mm');
@@ -1322,6 +1325,8 @@ t	CREATE TABLE `t` (
 ALTER TABLE t ADD COLUMN i INT GENERATED ALWAYS AS(a+a+b), ADD COLUMN j INT, ALGORITHM=INPLACE;
 ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: INPLACE ADD or DROP of virtual columns cannot be combined with other ALTER TABLE actions. Try ALGORITHM=COPY
 ALTER TABLE t ADD INDEX (x), ADD COLUMN j INT, ALGORITHM=INPLACE, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t ADD INDEX (x), ADD COLUMN j INT, ALGORITHM=INPLACE;
 SHOW CREATE TABLE t;
 Table	Create Table
 t	CREATE TABLE `t` (

mysql-test/suite/gcol/r/innodb_virtual_debug.result

@@ -53,46 +53,8 @@ a	b	c	h
 18	1	19	mm
 28	1	29	mm
 NULL	NULL	NULL	mx
-SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_create WAIT_FOR go_ahead';
-ALTER TABLE t ADD COLUMN x INT;
-connection con1;
-SET DEBUG_SYNC = 'now WAIT_FOR start_create';
-start transaction;
-update t set a=1 where a = 0;
-rollback;
-start transaction;
-delete from t;
-insert into t values(1,null,default,null);
-rollback;
-start transaction;
-update t set b=b+1;
-rollback;
-SET DEBUG_SYNC = 'now SIGNAL go_ahead';
-connection default;
-check table t;
-Table	Op	Msg_type	Msg_text
-test.t	check	status	OK
-SELECT c FROM t;
-c
-NULL
-3
-19
-29
-SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_create WAIT_FOR go_ahead';
-ALTER TABLE t ADD COLUMN x2 INT;
-connection con1;
-SET DEBUG_SYNC = 'now WAIT_FOR start_create';
-start transaction;
-DELETE FROM t WHERE a = 0;
-ROLLBACK;
-DELETE FROM t WHERE a = 0;
-SET DEBUG_SYNC = 'now SIGNAL go_ahead';
-connection default;
-SELECT c FROM t;
-c
-NULL
-19
-29
+ALTER TABLE t FORCE, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
 disconnect con1;
 DROP TABLE t;
 SET DEBUG_SYNC = 'RESET';
@@ -123,135 +85,4 @@ NULL
 19
 29
 DROP TABLE t;
-#
-# Bug#22018532 ASSERTION WHEN ONLINE REAPPLY REBUILD LOG ON
-# MULTIPLE INDEXED VIRTUAL COLUMNS
-#
-create table t (
-a int as (1) virtual,
-b int,
-c int as (1) virtual,
-unique(b),
-unique(c),
-key(a)
-) engine=innodb;
-insert ignore into t values();
-SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_create WAIT_FOR go_ahead';
-optimize table t;
-connect  con1,localhost,root,,;
-SET DEBUG_SYNC = 'now WAIT_FOR start_create';
-insert ignore into t values();
-Warnings:
-Warning	1062	Duplicate entry '1' for key 'c'
-SET DEBUG_SYNC = 'now SIGNAL go_ahead';
-connection default;
-/* connection default */ optimize table t;
-Table	Op	Msg_type	Msg_text
-test.t	optimize	note	Table does not support optimize, doing recreate + analyze instead
-test.t	optimize	error	Duplicate entry '1' for key 'a'
-test.t	optimize	status	Operation failed
-Warnings:
-Error	1062	Duplicate entry '1' for key 'a'
-SELECT c FROM t;
-c
-1
-SHOW CREATE TABLE t;
-Table	Create Table
-t	CREATE TABLE `t` (
-  `a` int(11) GENERATED ALWAYS AS (1) VIRTUAL,
-  `b` int(11) DEFAULT NULL,
-  `c` int(11) GENERATED ALWAYS AS (1) VIRTUAL,
-  UNIQUE KEY `b` (`b`),
-  UNIQUE KEY `c` (`c`),
-  KEY `a` (`a`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-SELECT * FROM t;
-a	b	c
-1	NULL	1
-DROP TABLE t;
-CREATE TABLE t (a INT, b INT, c INT GENERATED ALWAYS AS(a+b), h VARCHAR(10));
-INSERT INTO t VALUES (11, 3, DEFAULT, 'mm');
-INSERT INTO t VALUES (18, 1, DEFAULT, 'mm');
-INSERT INTO t VALUES (28, 1, DEFAULT, 'mm');
-INSERT INTO t VALUES (null, null, DEFAULT, 'mm');
-CREATE INDEX idx ON t(c);
-SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_rebuild WAIT_FOR go_ahead';
-optimize table t;
-connection con1;
-SET DEBUG_SYNC = 'now WAIT_FOR start_rebuild';
-INSERT INTO t VALUES (48, 2, DEFAULT, 'xx');
-INSERT INTO t VALUES (68, 3, DEFAULT, 'sx');
-SET DEBUG_SYNC = 'now SIGNAL go_ahead';
-connection default;
-/* connection default */ optimize table t;
-Table	Op	Msg_type	Msg_text
-test.t	optimize	note	Table does not support optimize, doing recreate + analyze instead
-test.t	optimize	status	OK
-SELECT c FROM t;
-c
-NULL
-14
-19
-29
-50
-71
-disconnect con1;
-DROP TABLE t;
-#
-#  Bug#22951879 - ASSERTS RELATED TO ONLINE DDL AND GCOL
-#
-create table ibstd_14 (a int not null, d int not null, b varchar(198) not null, c char(181), vadcol int as (a+length(d)) stored, vbcol char(2) as (substr(b,2,2)) virtual, vbidxcol char(3) as (substr(b,1,3)) virtual , index(d), index(a), index(vbidxcol), index(a,vbidxcol), index(vbidxcol,d), unique key (b(10), a, d), index(c(99), b(31)), index(b(5), c(10), a) , index(a,d)) engine=InnoDB stats_persistent=1 row_format=dynamic;
-SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_create WAIT_FOR go_ahead';
-alter table ibstd_14  row_format=compressed key_block_size=4,add  key kn3 (d,c,vbcol,b);
-connect  con1,localhost,root;
-SET DEBUG_SYNC = 'now WAIT_FOR start_create';
-insert into ibstd_14 (a,d,b,c, vbidxcol, vbcol) values ('118','6',repeat('oacolaarlruoacuroauurloraarucoooarcooauoolacalllaulrruarrrucruuooclacuoouccarrcoocloccorrrrarourcooalloocooccouruolaorlcaocualolc','1'),repeat('lolrrlalcocroraaulauclaaucolcorcuooaolruaooooluooooouaoorlarucorullalcrrloccououaooaorluorraclrcooouuolocoaolcocaaculruoocucoocoooauuolarcoraraocaoolulolarru','1'),default,default);
-insert into ibstd_14 (a,d,b,c, vbidxcol, vbcol) values ('118','6', 'aaaa', 'lll', default, default);
-update ibstd_14 set b='11111' where b='aaaa';
-SET DEBUG_SYNC = 'now SIGNAL go_ahead';
-connection default;
-select * from ibstd_14;
-a	d	b	c	vadcol	vbcol	vbidxcol
-118	6	oacolaarlruoacuroauurloraarucoooarcooauoolacalllaulrruarrrucruuooclacuoouccarrcoocloccorrrrarourcooalloocooccouruolaorlcaocualolc	lolrrlalcocroraaulauclaaucolcorcuooaolruaooooluooooouaoorlarucorullalcrrloccououaooaorluorraclrcooouuolocoaolcocaaculruoocucoocoooauuolarcoraraocaoolulolarru	119	ac	oac
-118	6	11111	lll	119	11	111
-select d,c,vbcol,b from  ibstd_14;
-d	c	vbcol	b
-6	lll	11	11111
-6	lolrrlalcocroraaulauclaaucolcorcuooaolruaooooluooooouaoorlarucorullalcrrloccououaooaorluorraclrcooouuolocoaolcocaaculruoocucoocoooauuolarcoraraocaoolulolarru	ac	oacolaarlruoacuroauurloraarucoooarcooauoolacalllaulrruarrrucruuooclacuoouccarrcoocloccorrrrarourcooalloocooccouruolaorlcaocualolc
-select vbcol from ibstd_14;
-vbcol
-11
-ac
-drop table ibstd_14;
-#
-# Bug#22018745 CORRUPTION IN ONLINE TABLE REBUILD
-# (ROW_FORMAT=REDUNDANT, INDEXED VIRTUAL COLUMN)
-#
-CREATE TABLE t (
-b char(5) PRIMARY KEY,
-v char(3) GENERATED ALWAYS AS (substr(b,1,3)) VIRTUAL, KEY(v)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT;
-connection con1;
-SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL prepared WAIT_FOR apply';
-OPTIMIZE TABLE t;
-connection default;
-SET DEBUG_SYNC='now WAIT_FOR prepared';
-INSERT INTO t SET b='fubar';
-BEGIN;
-DELETE FROM t;
-ROLLBACK;
-SET DEBUG_SYNC='now SIGNAL apply';
-connection con1;
-Table	Op	Msg_type	Msg_text
-test.t	optimize	note	Table does not support optimize, doing recreate + analyze instead
-test.t	optimize	status	OK
-connection default;
-CHECK TABLE t;
-Table	Op	Msg_type	Msg_text
-test.t	check	status	OK
-SELECT * FROM t;
-b	v
-fubar	fub
-DROP TABLE t;
-disconnect con1;
 SET DEBUG_SYNC = 'RESET';

mysql-test/suite/gcol/r/innodb_virtual_rebuild.result

@@ -0,0 +1,106 @@
+CREATE TABLE t1 (j SERIAL, i INT, v INT AS (i) VIRTUAL) ENGINE=InnoDB;
+CREATE TABLE t2 (j SERIAL, i INT, v INT AS (i) VIRTUAL) ENGINE=InnoDB
+ROW_FORMAT=REDUNDANT;
+CREATE TABLE t3 (i INT, v INT AS (i) VIRTUAL) ENGINE=InnoDB;
+CREATE TABLE t4 (i INT, v INT AS (i) VIRTUAL) ENGINE=InnoDB
+ROW_FORMAT=REDUNDANT;
+INSERT INTO t4 SET i=1;
+ALTER TABLE t4 ADD INDEX(v), LOCK=NONE;
+ALTER TABLE t4 ADD COLUMN k INT, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t4 ADD COLUMN k INT;
+ALTER TABLE t4 DROP k, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t4 DROP INDEX v, LOCK=NONE;
+ALTER TABLE t4 DROP k, LOCK=NONE;
+INSERT INTO t3 SET i=1;
+ALTER TABLE t3 ADD INDEX(v), LOCK=NONE;
+ALTER TABLE t3 ADD COLUMN k INT, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t3 ADD COLUMN k INT;
+ALTER TABLE t3 DROP k, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t3 DROP INDEX v, LOCK=NONE;
+ALTER TABLE t3 DROP k, LOCK=NONE;
+INSERT INTO t2 SET i=1;
+ALTER TABLE t2 ADD INDEX(v), LOCK=NONE;
+ALTER TABLE t2 ADD COLUMN k INT, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t2 ADD COLUMN k INT;
+ALTER TABLE t2 DROP k, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t2 DROP INDEX v, LOCK=NONE;
+ALTER TABLE t2 DROP k, LOCK=NONE;
+INSERT INTO t1 SET i=1;
+ALTER TABLE t1 ADD INDEX(v), LOCK=NONE;
+ALTER TABLE t1 ADD COLUMN k INT, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t1 ADD COLUMN k INT;
+ALTER TABLE t1 DROP k, LOCK=NONE;
+ERROR 0A000: LOCK=NONE is not supported. Reason: online rebuild with indexed virtual columns. Try LOCK=SHARED
+ALTER TABLE t1 DROP INDEX v, LOCK=NONE;
+ALTER TABLE t1 DROP k, LOCK=NONE;
+connect  ddl,localhost,root,,test;
+connection default;
+connection ddl;
+SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL copied WAIT_FOR dml';
+ALTER TABLE t4 FORCE;
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+BEGIN;
+UPDATE t4 SET i = 0;
+ROLLBACK;
+SET DEBUG_SYNC = 'now SIGNAL dml';
+connection ddl;
+connection default;
+SELECT * FROM t4;
+i	v
+1	1
+DROP TABLE t4;
+connection ddl;
+SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL copied WAIT_FOR dml';
+ALTER TABLE t3 FORCE;
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+BEGIN;
+UPDATE t3 SET i = 0;
+ROLLBACK;
+SET DEBUG_SYNC = 'now SIGNAL dml';
+connection ddl;
+connection default;
+SELECT * FROM t3;
+i	v
+1	1
+DROP TABLE t3;
+connection ddl;
+SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL copied WAIT_FOR dml';
+ALTER TABLE t2 FORCE;
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+BEGIN;
+UPDATE t2 SET i = 0;
+ROLLBACK;
+SET DEBUG_SYNC = 'now SIGNAL dml';
+connection ddl;
+connection default;
+SELECT * FROM t2;
+j	i	v
+1	1	1
+DROP TABLE t2;
+connection ddl;
+SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL copied WAIT_FOR dml';
+ALTER TABLE t1 FORCE;
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+BEGIN;
+UPDATE t1 SET i = 0;
+ROLLBACK;
+SET DEBUG_SYNC = 'now SIGNAL dml';
+connection ddl;
+connection default;
+SELECT * FROM t1;
+j	i	v
+1	1	1
+DROP TABLE t1;
+disconnect ddl;
+SET DEBUG_SYNC = 'RESET';

mysql-test/suite/gcol/t/innodb_virtual_basic.test

@@ -877,8 +877,13 @@ INSERT INTO t1 VALUES (18, 1, DEFAULT, 'mm');
 
 INSERT INTO t1 VALUES (28, 1, DEFAULT, 'mm');
 
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 ALTER TABLE t1 ADD INDEX idx12 (c) , FORCE, LOCK=NONE;
+ALTER TABLE t1 ADD INDEX idx12 (c), LOCK=NONE;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 ALTER TABLE t1 DROP COLUMN h,  ADD INDEX idx (c) , FORCE, LOCK=NONE;
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t1 DROP COLUMN h,  ADD INDEX idx (c), LOCK=NONE;
 
 DROP TABLE t1 ;
 
@@ -1304,8 +1309,9 @@ SHOW CREATE TABLE t;
 --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 ALTER TABLE t ADD COLUMN i INT GENERATED ALWAYS AS(a+a+b), ADD COLUMN j INT, ALGORITHM=INPLACE;
 
-# Add an index along with adding a regular column is allowed.
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 ALTER TABLE t ADD INDEX (x), ADD COLUMN j INT, ALGORITHM=INPLACE, LOCK=NONE;
+ALTER TABLE t ADD INDEX (x), ADD COLUMN j INT, ALGORITHM=INPLACE;
 SHOW CREATE TABLE t;
 
 # Online add an index on newly added virtual column is not allowed.

mysql-test/suite/gcol/t/innodb_virtual_debug.test

@@ -53,8 +53,11 @@ SELECT c FROM t;
 SHOW CREATE TABLE t;
 SELECT * FROM t;
 
+--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
+ALTER TABLE t FORCE, LOCK=NONE;
+if (0) {# MDEV-14341 TODO: re-enable this
 SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_create WAIT_FOR go_ahead';
---send ALTER TABLE t ADD COLUMN x INT
+--send ALTER TABLE t FORCE
 
 connection con1;
 
@@ -78,7 +81,7 @@ check table t;
 SELECT c FROM t;
 
 SET DEBUG_SYNC = 'innodb_inplace_alter_table_enter SIGNAL start_create WAIT_FOR go_ahead';
---send ALTER TABLE t ADD COLUMN x2 INT
+--send ALTER TABLE t FORCE
 
 connection con1;
 
@@ -93,6 +96,7 @@ connection default;
 reap;
 
 SELECT c FROM t;
+}
 
 disconnect con1;
 DROP TABLE t;
@@ -129,7 +133,7 @@ SELECT c FROM t;
 
 DROP TABLE t;
 
-
+if (0) {# MDEV-14341 TODO: re-enable LOCK=NONE and these tests
 --echo #
 --echo # Bug#22018532 ASSERTION WHEN ONLINE REAPPLY REBUILD LOG ON
 --echo # MULTIPLE INDEXED VIRTUAL COLUMNS
@@ -264,7 +268,7 @@ SELECT * FROM t;
 DROP TABLE t;
 
 disconnect con1;
-
+}
 SET DEBUG_SYNC = 'RESET';
 
 --source include/wait_until_count_sessions.inc