MDEV-16131 Assertion failed in dict_index_t::instant_field_value()

During a table-rebuilding online ALTER TABLE, if
dict_index_t::remove_instant() was invoked on the source table
(because it became empty), we would inadvertently change the way
how log records are written and parsed. We must keep the online_log
format unchanged throughout the whole table-rebuilding operation.

dict_col_t::def_t: Name the type of dict_col_t::def_val.

rec_get_n_add_field_len(), rec_set_n_add_field(): Define globally,
because these will be needed in row_log_table_low().

rec_init_offsets_temp(), rec_init_offsets_comp_ordinary(): Add
the parameter def_val for explicitly passing the default values
of the instantly added columns of the source table, so that
dict_index_t::instant_field_value() will not be called during
row_log_table_apply(). This allows us to consistently parse the
online_log records, even if the source table was converted
to the canonical non-instant format during the rebuild operation.

row_log_t::non_core_fields[]: The default values of the
instantly added columns on the source table; copied
during ha_innobase::prepare_inplace_alter_table()
while the table is exclusively locked.

row_log_t::instant_field_value(): Accessor to non_core_fields[],
analogous to dict_index_t::instant_field_value().

row_log_table_low(): Add fake_extra_size bytes to the record
header if the source table was converted to the canonical format
during the operation.

row_log_allocate(): Initialize row_log_t::non_core_fields.

mysql-test/suite/innodb/r/instant_alter_debug.result

@@ -175,10 +175,42 @@ SET DEBUG_SYNC='now WAIT_FOR copied';
 BEGIN;
 INSERT INTO t1 SET b=1;
 ROLLBACK;
-disconnect stop_purge;
+connection stop_purge;
+COMMIT;
+connection default;
 InnoDB		2 transactions not purged
 SET DEBUG_SYNC='now SIGNAL logged';
-disconnect ddl;
+connection ddl;
+connection default;
 DROP TABLE t1;
 SET DEBUG_SYNC='RESET';
+#
+# MDEV-16131 Assertion failed in dict_index_t::instant_field_value()
+#
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 SET a=0;
+ALTER TABLE t1 ADD COLUMN b INT NOT NULL DEFAULT 2, ADD COLUMN c INT;
+connection stop_purge;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+connection default;
+DELETE FROM t1;
+connection ddl;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL copied WAIT_FOR logged';
+ALTER TABLE t1 FORCE;
+disconnect stop_purge;
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+InnoDB		1 transactions not purged
+INSERT INTO t1 SET a=1;
+INSERT INTO t1 SET a=2,b=3,c=4;
+SET DEBUG_SYNC = 'now SIGNAL logged';
+connection ddl;
+disconnect ddl;
+connection default;
+SET DEBUG_SYNC = RESET;
+SELECT * FROM t1;
+a	b	c
+1	2	NULL
+2	3	4
+DROP TABLE t1;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;

mysql-test/suite/innodb/t/instant_alter_debug.test

@@ -189,16 +189,56 @@ SET DEBUG_SYNC='now WAIT_FOR copied';
 BEGIN;
 INSERT INTO t1 SET b=1;
 ROLLBACK;
-disconnect stop_purge;
+connection stop_purge;
+COMMIT;
+connection default;
 
 # Wait for purge to empty the table.
 let $wait_all_purged=2;
 --source include/wait_all_purged.inc
 let $wait_all_purged=0;
 
 SET DEBUG_SYNC='now SIGNAL logged';
-disconnect ddl;
+connection ddl;
+reap;
+connection default;
 DROP TABLE t1;
 SET DEBUG_SYNC='RESET';
 
+--echo #
+--echo # MDEV-16131 Assertion failed in dict_index_t::instant_field_value()
+--echo #
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 SET a=0;
+ALTER TABLE t1 ADD COLUMN b INT NOT NULL DEFAULT 2, ADD COLUMN c INT;
+
+connection stop_purge;
+START TRANSACTION WITH CONSISTENT SNAPSHOT;
+
+connection default;
+DELETE FROM t1;
+
+connection ddl;
+SET DEBUG_SYNC='row_log_table_apply1_before SIGNAL copied WAIT_FOR logged';
+send ALTER TABLE t1 FORCE;
+
+disconnect stop_purge;
+
+connection default;
+SET DEBUG_SYNC = 'now WAIT_FOR copied';
+let $wait_all_purged = 1;
+--source include/wait_all_purged.inc
+INSERT INTO t1 SET a=1;
+INSERT INTO t1 SET a=2,b=3,c=4;
+SET DEBUG_SYNC = 'now SIGNAL logged';
+
+connection ddl;
+reap;
+disconnect ddl;
+
+connection default;
+SET DEBUG_SYNC = RESET;
+SELECT * FROM t1;
+DROP TABLE t1;
+
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;

storage/innobase/include/dict0mem.h

@@ -593,7 +593,7 @@ struct dict_col_t{
 	inline void detach(const dict_index_t& index);
 
 	/** Data for instantly added columns */
-	struct {
+	struct def_t {
 		/** original default value of instantly added column */
 		const void*	data;
 		/** len of data, or UNIV_SQL_DEFAULT if unavailable */

storage/innobase/include/rem0rec.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -312,6 +312,31 @@ rec_set_status(rec_t* rec, byte bits)
 		| bits;
 }
 
+/** Get the length of added field count in a REC_STATUS_COLUMNS_ADDED record.
+@param[in]	n_add_field	number of added fields, minus one
+@return	storage size of the field count, in bytes */
+inline unsigned rec_get_n_add_field_len(ulint n_add_field)
+{
+	ut_ad(n_add_field < REC_MAX_N_FIELDS);
+	return n_add_field < 0x80 ? 1 : 2;
+}
+
+/** Set the added field count in a REC_STATUS_COLUMNS_ADDED record.
+@param[in,out]	header	variable header of a REC_STATUS_COLUMNS_ADDED record
+@param[in]	n_add	number of added fields, minus 1
+@return	record header before the number of added fields */
+inline void rec_set_n_add_field(byte*& header, ulint n_add)
+{
+	ut_ad(n_add < REC_MAX_N_FIELDS);
+
+	if (n_add < 0x80) {
+		*header-- = byte(n_add);
+	} else {
+		*header-- = byte(n_add) | 0x80;
+		*header-- = byte(n_add >> 7);
+	}
+}
+
 /******************************************************//**
 The following function is used to retrieve the info and status
 bits of a record.  (Only compact records have status bits.)
@@ -962,13 +987,15 @@ rec_get_converted_size_temp(
 @param[in]	index	index of that the record belongs to
 @param[in,out]	offsets	offsets to the fields; in: rec_offs_n_fields(offsets)
 @param[in]	n_core	number of core fields (index->n_core_fields)
+@param[in]	def_val	default values for non-core fields
 @param[in]	status	REC_STATUS_ORDINARY or REC_STATUS_COLUMNS_ADDED */
 void
 rec_init_offsets_temp(
 	const rec_t*		rec,
 	const dict_index_t*	index,
 	ulint*			offsets,
 	ulint			n_core,
+	const dict_col_t::def_t*def_val,
 	rec_comp_status_t	status = REC_STATUS_ORDINARY)
 	MY_ATTRIBUTE((nonnull));
 /** Determine the offset to each field in temporary file.

storage/innobase/rem/rem0rec.cc

@@ -237,15 +237,6 @@ rec_get_n_extern_new(
 	return(n_extern);
 }
 
-/** Get the length of added field count in a REC_STATUS_COLUMNS_ADDED record.
-@param[in]	n_add_field	number of added fields, minus one
-@return	storage size of the field count, in bytes */
-static inline unsigned rec_get_n_add_field_len(ulint n_add_field)
-{
-	ut_ad(n_add_field < REC_MAX_N_FIELDS);
-	return n_add_field < 0x80 ? 1 : 2;
-}
-
 /** Get the added field count in a REC_STATUS_COLUMNS_ADDED record.
 @param[in,out]	header	variable header of a REC_STATUS_COLUMNS_ADDED record
 @return	number of added fields */
@@ -264,22 +255,6 @@ static inline unsigned rec_get_n_add_field(const byte*& header)
 	return n_fields_add;
 }
 
-/** Set the added field count in a REC_STATUS_COLUMNS_ADDED record.
-@param[in,out]	header	variable header of a REC_STATUS_COLUMNS_ADDED record
-@param[in]	n_add	number of added fields, minus 1
-@return	record header before the number of added fields */
-static inline void rec_set_n_add_field(byte*& header, ulint n_add)
-{
-	ut_ad(n_add < REC_MAX_N_FIELDS);
-
-	if (n_add < 0x80) {
-		*header-- = byte(n_add);
-	} else {
-		*header-- = byte(n_add) | 0x80;
-		*header-- = byte(n_add >> 7);
-	}
-}
-
 /** Format of a leaf-page ROW_FORMAT!=REDUNDANT record */
 enum rec_leaf_format {
 	/** Temporary file record */
@@ -299,6 +274,8 @@ This is a special case of rec_init_offsets() and rec_get_offsets_func().
 @param[in]	rec	leaf-page record
 @param[in]	index	the index that the record belongs in
 @param[in]	n_core	number of core fields (index->n_core_fields)
+@param[in]	def_val	default values for non-core fields, or
+			NULL to refer to index->fields[].col->def_val
 @param[in,out]	offsets	offsets, with valid rec_offs_n_fields(offsets)
 @param[in]	format	record format */
 static inline
@@ -308,6 +285,7 @@ rec_init_offsets_comp_ordinary(
 	const dict_index_t*	index,
 	ulint*			offsets,
 	ulint			n_core,
+	const dict_col_t::def_t*def_val,
 	rec_leaf_format		format)
 {
 	ulint		offs		= 0;
@@ -379,7 +357,19 @@ rec_init_offsets_comp_ordinary(
 		ulint			len;
 
 		/* set default value flag */
-		if (i >= n_fields) {
+		if (i < n_fields) {
+		} else if (def_val) {
+			const dict_col_t::def_t& d = def_val[i - n_core];
+			if (!d.data) {
+				len = offs | REC_OFFS_SQL_NULL;
+				ut_ad(d.len == UNIV_SQL_NULL);
+			} else {
+				len = offs | REC_OFFS_DEFAULT;
+				any |= REC_OFFS_DEFAULT;
+			}
+
+			goto resolved;
+		} else {
 			ulint dlen;
 			if (!index->instant_field_value(i, &dlen)) {
 				len = offs | REC_OFFS_SQL_NULL;
@@ -618,12 +608,14 @@ rec_init_offsets(
 			ut_ad(leaf);
 			rec_init_offsets_comp_ordinary(rec, index, offsets,
 						       index->n_core_fields,
+						       NULL,
 						       REC_LEAF_COLUMNS_ADDED);
 			return;
 		case REC_STATUS_ORDINARY:
 			ut_ad(leaf);
 			rec_init_offsets_comp_ordinary(rec, index, offsets,
 						       index->n_core_fields,
+						       NULL,
 						       REC_LEAF_ORDINARY);
 			return;
 		}
@@ -1695,13 +1687,15 @@ rec_get_converted_size_temp(
 @param[in]	index	index of that the record belongs to
 @param[in,out]	offsets	offsets to the fields; in: rec_offs_n_fields(offsets)
 @param[in]	n_core	number of core fields (index->n_core_fields)
+@param[in]	def_val	default values for non-core fields
 @param[in]	status	REC_STATUS_ORDINARY or REC_STATUS_COLUMNS_ADDED */
 void
 rec_init_offsets_temp(
 	const rec_t*		rec,
 	const dict_index_t*	index,
 	ulint*			offsets,
 	ulint			n_core,
+	const dict_col_t::def_t*def_val,
 	rec_comp_status_t	status)
 {
 	ut_ad(status == REC_STATUS_ORDINARY
@@ -1710,7 +1704,7 @@ rec_init_offsets_temp(
 	if it was emptied during an ALTER TABLE operation. */
 	ut_ad(index->n_core_fields == n_core || !index->is_instant());
 	ut_ad(index->n_core_fields >= n_core);
-	rec_init_offsets_comp_ordinary(rec, index, offsets, n_core,
+	rec_init_offsets_comp_ordinary(rec, index, offsets, n_core, def_val,
 				       status == REC_STATUS_COLUMNS_ADDED
 				       ? REC_LEAF_TEMP_COLUMNS_ADDED
 				       : REC_LEAF_TEMP);
@@ -1729,7 +1723,8 @@ rec_init_offsets_temp(
 {
 	ut_ad(!index->is_instant());
 	rec_init_offsets_comp_ordinary(rec, index, offsets,
-				       index->n_core_fields, REC_LEAF_TEMP);
+				       index->n_core_fields, NULL,
+				       REC_LEAF_TEMP);
 }
 
 /** Convert a data tuple prefix to the temporary file format.