MDEV-15250 UPSERT during ALTER TABLE results in 'Duplicate entry' error for alter

- InnoDB DDL results in `Duplicate entry' if concurrent DML throws
duplicate key error. The following scenario explains the problem

connection con1:
  ALTER TABLE t1 FORCE;

connection con2:
  INSERT INTO t1(pk, uk) VALUES (2, 2), (3, 2);

In connection con2, InnoDB throws the 'DUPLICATE KEY' error because
of unique index. Alter operation will throw the error when applying
the concurrent DML log.

- Inserting the duplicate key for unique index logs the insert
operation for online ALTER TABLE. When insertion fails,
transaction does rollback and it leads to logging of
delete operation for online ALTER TABLE.
While applying the insert log entries, alter operation
encounters 'DUPLICATE KEY' error.

- To avoid the above fake duplicate scenario, InnoDB should
not write any log for online ALTER TABLE before DML transaction
commit.

- User thread which does DML can apply the online log if
InnoDB ran out of online log and index is marked as completed.
Set online log error if apply phase encountered any error.
It can also clear all other indexes log, marks the newly
added indexes as corrupted.

- Removed the old online code which was a part of DML operations

commit_inplace_alter_table() : Does apply the online log
for the last batch of secondary index log and does frees
the log for the completed index.

trx_t::apply_online_log: Set to true while writing the undo
log if the modified table has active DDL

trx_t::apply_log(): Apply the DML changes to online DDL tables

dict_table_t::is_active_ddl(): Returns true if the table
has an active DDL

dict_index_t::online_log_make_dummy(): Assign dummy value
for clustered index online log to indicate the secondary
indexes are being rebuild.

dict_index_t::online_log_is_dummy(): Check whether the online
log has dummy value

ha_innobase_inplace_ctx::log_failure(): Handle the apply log
failure for online DDL transaction

row_log_mark_other_online_index_abort(): Clear out all other
online index log after encountering the error during
row_log_apply()

row_log_get_error(): Get the error happened during row_log_apply()

row_log_online_op(): Does apply the online log if index is
completed and ran out of memory. Returns false if apply log fails

UndorecApplier: Introduced a class to maintain the undo log
record, latched undo buffer page, parse the undo log record,
maintain the undo record type, info bits and update vector

UndorecApplier::get_old_rec(): Get the correct version of the
clustered index record that was modified by the current undo
log record

UndorecApplier::clear_undo_rec(): Clear the undo log related
information after applying the undo log record

UndorecApplier::log_update(): Handle the update, delete undo
log and apply it on online indexes

UndorecApplier::log_insert(): Handle the insert undo log
and apply it on online indexes

UndorecApplier::is_same(): Check whether the given roll pointer
is generated by the current undo log record information

trx_t::rollback_low(): Set apply_online_log for the transaction
after partially rollbacked transaction has any active DDL

prepare_inplace_alter_table_dict(): After allocating the online
log, InnoDB does create fulltext common tables. Fulltext index
doesn't allow the index to be online. So removed the dead
code of online log removal

Thanks to Marko Mäkelä for providing the initial prototype and
Matthias Leich for testing the issue patiently.

Author: Thirunarayanan

include/my_compiler.h

@@ -2,7 +2,7 @@
 #define MY_COMPILER_INCLUDED
 
 /* Copyright (c) 2010, 2011, Oracle and/or its affiliates. All rights reserved.
-   Copyright (c) 2017, 2020, MariaDB Corporation.
+   Copyright (c) 2017, 2022, MariaDB Corporation.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -39,15 +39,8 @@
 
 /* GNU C/C++ */
 #if defined __GNUC__
-/* Convenience macro to test the minimum required GCC version. */
-# define MY_GNUC_PREREQ(maj, min) \
-    ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
-/* Any after 2.95... */
 # define MY_ALIGN_EXT
-/* Comunicate to the compiler the unreachability of the code. */
-# if MY_GNUC_PREREQ(4,5)
-#   define MY_ASSERT_UNREACHABLE()   __builtin_unreachable()
-# endif
+# define MY_ASSERT_UNREACHABLE()   __builtin_unreachable()
 
 /* Microsoft Visual C++ */
 #elif defined _MSC_VER
@@ -158,7 +151,6 @@ struct my_aligned_storage
 #ifdef __GNUC__
 # define ATTRIBUTE_NORETURN __attribute__((noreturn))
 # define ATTRIBUTE_NOINLINE __attribute__((noinline))
-# if MY_GNUC_PREREQ(4,3)
 /** Starting with GCC 4.3, the "cold" attribute is used to inform the
 compiler that a function is unlikely executed.  The function is
 optimized for size rather than speed and on many targets it is placed
@@ -167,8 +159,7 @@ appears close together improving code locality of non-cold parts of
 program.  The paths leading to call of cold functions within code are
 marked as unlikely by the branch prediction mechanism.  optimize a
 rarely invoked function for size instead for speed. */
-#  define ATTRIBUTE_COLD __attribute__((cold))
-# endif
+# define ATTRIBUTE_COLD __attribute__((cold))
 #elif defined _MSC_VER
 # define ATTRIBUTE_NORETURN __declspec(noreturn)
 # define ATTRIBUTE_NOINLINE __declspec(noinline)

mysql-test/suite/gcol/r/innodb_virtual_debug.result

@@ -105,7 +105,7 @@ SET lock_wait_timeout = 1;
 ALTER TABLE t1 ADD UNIQUE INDEX(c, b);
 connection default;
 SET DEBUG_SYNC = 'now WAIT_FOR s1';
-SET DEBUG_SYNC = 'row_ins_sec_index_enter SIGNAL s2 WAIT_FOR s3';
+SET DEBUG_SYNC = 'row_log_insert_handle SIGNAL s2 WAIT_FOR s3';
 INSERT INTO t1(a, b) VALUES(2, 2);
 connection con1;
 ERROR HY000: Lock wait timeout exceeded; try restarting transaction

mysql-test/suite/gcol/t/innodb_virtual_debug.test

@@ -295,7 +295,7 @@ SET lock_wait_timeout = 1;
 
 connection default;
 SET DEBUG_SYNC = 'now WAIT_FOR s1';
-SET DEBUG_SYNC = 'row_ins_sec_index_enter SIGNAL s2 WAIT_FOR s3';
+SET DEBUG_SYNC = 'row_log_insert_handle SIGNAL s2 WAIT_FOR s3';
 --send INSERT INTO t1(a, b) VALUES(2, 2)
 
 connection con1;

mysql-test/suite/innodb/r/alter_candidate_key.result

@@ -74,7 +74,7 @@ connection con1;
 SET DEBUG_SYNC='now WAIT_FOR dml';
 BEGIN;
 INSERT INTO t1 SET a=NULL;
-ROLLBACK;
+COMMIT;
 set DEBUG_SYNC='now SIGNAL dml_done';
 connection default;
 ERROR 22004: Invalid use of NULL value

mysql-test/suite/innodb/r/alter_crash.result

@@ -169,9 +169,6 @@ INSERT INTO t1(f1, f2) VALUES(2, "This is column2 value");
 ROLLBACK;
 set DEBUG_SYNC = 'now SIGNAL insert_done';
 connection default;
-Warnings:
-Warning	1265	Data truncated for column 'f3' at row 3
-Warning	1265	Data truncated for column 'f4' at row 3
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -202,6 +199,7 @@ connection default;
 SET DEBUG_SYNC = 'now WAIT_FOR scanned';
 BEGIN;
 INSERT INTO t1 VALUES(2,1);
+COMMIT;
 SET DEBUG_SYNC = 'now SIGNAL commit';
 SET DEBUG_SYNC = 'now WAIT_FOR c';
 SET GLOBAL innodb_fil_make_page_dirty_debug=0;
@@ -221,4 +219,5 @@ t1	CREATE TABLE `t1` (
 SELECT * FROM t1;
 a	b
 1	1
+2	1
 DROP TABLE t1;

mysql-test/suite/innodb/r/alter_dml_apply.result

@@ -0,0 +1,39 @@
+CREATE TABLE t1(f1 INT NOT NULL, f2 INT NOT NULL,
+f3 CHAR(200), f4 CHAR(200),
+PRIMARY KEY(f1))ENGINE=InnoDB;
+INSERT INTO t1 VALUES(6000, 6000, "InnoDB", "MariaDB");
+SET DEBUG_SYNC="inplace_after_index_build SIGNAL dml_start WAIT_FOR dml_commit";
+ALTER TABLE t1 ADD UNIQUE KEY(f2), ADD UNIQUE INDEX(f4(10));
+connect con1,localhost,root,,,;
+SET DEBUG_SYNC="now WAIT_FOR dml_start";
+BEGIN;
+DELETE FROM t1 WHERE f1= 6000;
+INSERT INTO t1 VALUES(6000, 6000, "InnoDB", "MariaDB");
+ROLLBACK;
+BEGIN;
+DELETE FROM t1 WHERE f1= 6000;
+INSERT INTO t1 VALUES(6000, 6000, "InnoDB", "MariaDB");
+INSERT INTO t1 SELECT seq, seq, repeat('a', 200), repeat('b', 200) FROM seq_1_to_4000;
+COMMIT;
+SET DEBUG_SYNC="now SIGNAL dml_commit";
+connection default;
+ERROR 23000: Duplicate entry '' for key '*UNKNOWN*'
+SET DEBUG_SYNC="inplace_after_index_build SIGNAL dml_start WAIT_FOR dml_commit";
+ALTER TABLE t1 ADD UNIQUE KEY(f2), ADD INDEX(f3(10));
+connection con1;
+SET DEBUG_SYNC="now WAIT_FOR dml_start";
+BEGIN;
+DELETE FROM t1;
+INSERT INTO t1 SELECT seq, seq, repeat('d', 200), repeat('e', 200) FROM
+seq_1_to_4000;
+UPDATE t1 SET f3=repeat('c', 200), f4= repeat('d', 200), f2=3;
+COMMIT;
+SET DEBUG_SYNC="now SIGNAL dml_commit";
+connection default;
+ERROR 23000: Duplicate entry '' for key '*UNKNOWN*'
+disconnect con1;
+CHECK TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+DROP TABLE t1;
+SET DEBUG_SYNC=reset;

mysql-test/suite/innodb/r/alter_mdl_timeout.result

@@ -10,7 +10,7 @@ begin;
 INSERT INTO t1 VALUES('e','e',5, 5);
 SET DEBUG_SYNC="now SIGNAL con1_insert";
 SET DEBUG_SYNC="now WAIT_FOR con1_wait";
-SET DEBUG_SYNC="before_row_upd_sec_new_index_entry SIGNAL con1_update WAIT_FOR alter_rollback";
+SET DEBUG_SYNC="after_row_upd_clust SIGNAL con1_update WAIT_FOR alter_rollback";
 UPDATE t1 set f4 = 10 order by f1 desc limit 2;
 connection default;
 ERROR HY000: Lock wait timeout exceeded; try restarting transaction

mysql-test/suite/innodb/r/alter_not_null_debug,STRICT.rdiff

@@ -1,11 +1,16 @@
-18,21c18
-< affected rows: 0
-< info: Records: 0  Duplicates: 0  Warnings: 1
-< Warnings:
-< Warning	1265	Data truncated for column 'c2' at row 3
----
-> ERROR 01000: Data truncated for column 'c2' at row 3
-24c21
-< 2	0
----
-> 2	NULL
+@@ -15,13 +15,10 @@
+ SET DEBUG_SYNC= 'now SIGNAL flushed';
+ affected rows: 0
+ connection default;
+-affected rows: 0
+-info: Records: 0  Duplicates: 0  Warnings: 1
+-Warnings:
+-Warning	1265	Data truncated for column 'c2' at row 3
++ERROR 22004: Invalid use of NULL value
+ SELECT * FROM t1;
+ c1	c2
+-2	0
++2	NULL
+ 3	1
+ DROP TABLE t1;
+ CREATE TABLE t1(c1 INT NOT NULL, c2 INT, PRIMARY KEY(c1))ENGINE=INNODB;

mysql-test/suite/innodb/r/innodb-alter-debug.result

@@ -43,9 +43,8 @@ SET DEBUG_SYNC = 'now SIGNAL s2';
 /* connection default */
 connection default;
 /* reap */ alter table t1 force, add b int, ALGORITHM=inplace;
-ERROR 23000: Duplicate entry '1' for key 'uk'
 SET DEBUG_SYNC = 'row_log_table_apply1_before SIGNAL s1 WAIT_FOR s2';
-alter table t1 force, add b int, ALGORITHM=inplace;;
+alter table t1 force, add c int, ALGORITHM=inplace;;
 /* connection con1 */
 connection con1;
 set DEBUG_SYNC = 'now WAIT_FOR s1';
@@ -55,7 +54,6 @@ SET DEBUG_SYNC = 'now SIGNAL s2';
 /* connection default */
 connection default;
 /* reap */ alter table t1 force, add b int, ALGORITHM=inplace;
-ERROR 23000: Duplicate entry '1' for key 'uk'
 SET DEBUG_SYNC = 'RESET';
 drop table t1;
 #
@@ -72,7 +70,6 @@ ERROR 23000: Duplicate entry '1' for key 'a'
 SET DEBUG_SYNC = 'now SIGNAL S2';
 disconnect con1;
 connection default;
-ERROR 23000: Duplicate entry '1' for key 'a'
 SET DEBUG_SYNC='RESET';
 DROP TABLE t1;
 #

mysql-test/suite/innodb/r/innodb-index-debug.result

@@ -118,20 +118,21 @@ drop table t480;
 # MDEV-12827 Assertion failure when reporting duplicate key error
 # in online table rebuild
 #
-CREATE TABLE t1 (j INT UNIQUE, i INT UNIQUE) ENGINE=InnoDB;
+CREATE TABLE t1 (j INT UNIQUE, i INT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(2, 2);
 connect  con1,localhost,root,,test;
 SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL built WAIT_FOR log';
-ALTER TABLE t1 DROP j, FORCE;
+ALTER TABLE t1 DROP j, ADD UNIQUE INDEX(i), FORCE;
 connection default;
 SET DEBUG_SYNC='now WAIT_FOR built';
 SET DEBUG_DBUG='+d,row_ins_row_level';
 INSERT INTO t1 (i) VALUES (0),(0);
-ERROR 23000: Duplicate entry '0' for key 'i'
 SET DEBUG_SYNC='now SIGNAL log';
 SET DEBUG_DBUG=@saved_debug_dbug;
 connection con1;
 ERROR 23000: Duplicate entry '0' for key 'i'
+DELETE FROM t1;
+ALTER TABLE t1 ADD UNIQUE INDEX(i);
 SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL built2 WAIT_FOR log2';
 ALTER TABLE t1 DROP j, FORCE;
 connection default;
@@ -141,7 +142,6 @@ UPDATE t1 SET i=0;
 ERROR 23000: Duplicate entry '0' for key 'i'
 SET DEBUG_SYNC='now SIGNAL log2';
 connection con1;
-ERROR 23000: Duplicate entry '0' for key 'i'
 disconnect con1;
 connection default;
 SET DEBUG_SYNC='RESET';