MDEV-14717: Prevent crash-downgrade to earlier MariaDB 10.2

A crash-downgrade of a RENAME (or TRUNCATE or table-rebuilding
ALTER TABLE or OPTIMIZE TABLE) operation to an earlier 10.2 version
would trigger a debug assertion failure during rollback,
in trx_roll_pop_top_rec_of_trx(). In a non-debug build, the
TRX_UNDO_RENAME_TABLE record would be misinterpreted as an
update_undo log record, and typically the file name would be
interpreted as DB_TRX_ID,DB_ROLL_PTR,PRIMARY KEY. If a matching
record would be found, row_undo_mod() would hit ut_error in
switch (node->rec_type). Typically, ut_a(table2 == NULL) would
fail when opening the table from SQL.

Because of this, we prevent a crash-downgrade to earlier MariaDB 10.2
versions by changing the InnoDB redo log format identifier to the
10.3 identifier, and by introducing a subformat identifier so that
10.2 can continue to refuse crash-downgrade from 10.3 or later.
After a clean shutdown, a downgrade to MariaDB 10.2.13 or later would
still be possible thanks to MDEV-14909. A downgrade to older 10.2
versions is only possible after removing the log files (not recommended).

LOG_HEADER_FORMAT_CURRENT: Change to 103 (originally the 10.3 format).

log_group_t: Add subformat. For 10.2, we will use subformat 1,
and will refuse crash recovery from any other subformat of the
10.3 format, that is, a genuine 10.3 redo log.

recv_find_max_checkpoint(): Allow startup after clean shutdown
from a future LOG_HEADER_FORMAT_10_4 (unencrypted only).
We cannot handle the encrypted 10.4 redo log block format,
which was introduced in MDEV-12041. Allow crash recovery from
the original 10.2 format as well as the new format.
In Mariabackup --backup, do not allow any startup from 10.3 or 10.4
redo logs.

recv_recovery_from_checkpoint_start(): Skip redo log apply for
clean 10.3 redo log, but not for the new 10.2 redo log
(10.3 format, subformat 1).

srv_prepare_to_delete_redo_log_files(): On format or subformat
mismatch, set srv_log_file_size = 0, so that we will display the
correct message.

innobase_start_or_create_for_mysql(): Check for format or subformat
mismatch.

xtrabackup_backup_func(): Remove debug assertions that were made
redundant by the code changes in recv_find_max_checkpoint().

extra/mariabackup/xtrabackup.cc

@@ -4139,9 +4139,6 @@ xtrabackup_backup_func()
 		goto log_fail;
 	}
 
-	ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT)
-		& ~LOG_HEADER_FORMAT_ENCRYPTED));
-
 	const byte* buf = log_sys->checkpoint_buf;
 
 reread_log_header:
@@ -4158,9 +4155,6 @@ xtrabackup_backup_func()
 		goto old_format;
 	}
 
-	ut_ad(!((log_sys->log.format ^ LOG_HEADER_FORMAT_CURRENT)
-		& ~LOG_HEADER_FORMAT_ENCRYPTED));
-
 	log_group_header_read(&log_sys->log, max_cp_field);
 
 	if (checkpoint_no_start != mach_read_from_8(buf + LOG_CHECKPOINT_NO)) {
@@ -4188,7 +4182,8 @@ xtrabackup_backup_func()
 	byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr[OS_FILE_LOG_BLOCK_SIZE];
 	memset(log_hdr, 0, sizeof log_hdr);
 	mach_write_to_4(LOG_HEADER_FORMAT + log_hdr, log_sys->log.format);
-	mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr, 1);
+	mach_write_to_4(LOG_HEADER_SUBFORMAT + log_hdr,
+			log_sys->log.subformat);
 	mach_write_to_8(LOG_HEADER_START_LSN + log_hdr, checkpoint_lsn_start);
 	strcpy(reinterpret_cast<char*>(LOG_HEADER_CREATOR + log_hdr),
 	       "Backup " MYSQL_SERVER_VERSION);

storage/innobase/include/log0log.h

@@ -513,10 +513,17 @@ or the MySQL version that created the redo log file. */
 	IB_TO_STR(MYSQL_VERSION_PATCH)
 
 /** The redo log format identifier corresponding to the current format version.
-Stored in LOG_HEADER_FORMAT. */
-#define LOG_HEADER_FORMAT_CURRENT	1
-/** The MariaDB 10.3.2 log format */
-#define LOG_HEADER_FORMAT_10_3		103
+Stored in LOG_HEADER_FORMAT.
+To prevent crash-downgrade to earlier 10.2 due to the inability to
+roll back a retroactively introduced TRX_UNDO_RENAME_TABLE undo log record,
+MariaDB 10.2.18 and later will use the 10.3 format, but LOG_HEADER_SUBFORMAT
+1 instead of 0. MariaDB 10.3 will use subformat 0 (5.7-style TRUNCATE) or 2
+(MDEV-13564 backup-friendly TRUNCATE). */
+#define LOG_HEADER_FORMAT_CURRENT	103
+/** The old MariaDB 10.2.2..10.2.17 log format */
+#define LOG_HEADER_FORMAT_10_2		1
+/** Future MariaDB 10.4 log format */
+#define LOG_HEADER_FORMAT_10_4		104
 /** Encrypted MariaDB redo log */
 #define LOG_HEADER_FORMAT_ENCRYPTED	(1U<<31)
 
@@ -553,7 +560,10 @@ struct log_group_t{
 	/** number of files in the group */
 	ulint				n_files;
 	/** format of the redo log: e.g., LOG_HEADER_FORMAT_CURRENT */
-	ulint				format;
+	uint32_t			format;
+	/** redo log subformat: 0 with separately logged TRUNCATE,
+	1 with fully redo-logged TRUNCATE */
+	uint32_t			subformat;
 	/** individual log file size in bytes, including the header */
 	lsn_t				file_size;
 	/** corruption status */

storage/innobase/log/log0log.cc

@@ -793,6 +793,7 @@ log_init(ulint n_files)
 	group->format = srv_encrypt_log
 		? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED
 		: LOG_HEADER_FORMAT_CURRENT;
+	group->subformat = 1;
 	group->file_size = srv_log_file_size;
 	group->state = LOG_GROUP_OK;
 	group->lsn = LOG_START_LSN;
@@ -880,7 +881,7 @@ log_group_file_header_flush(
 
 	memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE);
 	mach_write_to_4(buf + LOG_HEADER_FORMAT, group->format);
-	mach_write_to_4(buf + LOG_HEADER_SUBFORMAT, 1);
+	mach_write_to_4(buf + LOG_HEADER_SUBFORMAT, group->subformat);
 	mach_write_to_8(buf + LOG_HEADER_START_LSN, start_lsn);
 	strcpy(reinterpret_cast<char*>(buf) + LOG_HEADER_CREATOR,
 	       LOG_HEADER_CREATOR_CURRENT);

storage/innobase/log/log0recv.cc

@@ -1147,6 +1147,9 @@ recv_find_max_checkpoint(ulint* max_field)
 	/* Check the header page checksum. There was no
 	checksum in the first redo log format (version 0). */
 	group->format = mach_read_from_4(buf + LOG_HEADER_FORMAT);
+	group->subformat = group->format
+		? mach_read_from_4(buf + LOG_HEADER_SUBFORMAT)
+		: 0;
 	if (group->format != 0
 	    && !recv_check_log_header_checksum(buf)) {
 		ib::error() << "Invalid redo log header checksum.";
@@ -1164,8 +1167,11 @@ recv_find_max_checkpoint(ulint* max_field)
 		return(recv_find_max_checkpoint_0(&group, max_field));
 	case LOG_HEADER_FORMAT_CURRENT:
 	case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED:
-	case LOG_HEADER_FORMAT_10_3:
-	case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED:
+	case LOG_HEADER_FORMAT_10_2:
+	case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED:
+	case LOG_HEADER_FORMAT_10_4:
+		/* We can only parse the unencrypted LOG_HEADER_FORMAT_10_4.
+		The encrypted format uses a larger redo log block trailer. */
 		break;
 	default:
 		ib::error() << "Unsupported redo log format."
@@ -1240,8 +1246,20 @@ recv_find_max_checkpoint(ulint* max_field)
 	}
 
 	switch (group->format) {
-	case LOG_HEADER_FORMAT_10_3:
-	case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED:
+	case LOG_HEADER_FORMAT_CURRENT:
+	case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED:
+		if (group->subformat == 1) {
+			/* 10.2 with new crash-safe TRUNCATE */
+			break;
+		}
+		/* fall through */
+	case LOG_HEADER_FORMAT_10_4:
+		if (srv_operation == SRV_OPERATION_BACKUP) {
+			ib::error()
+				<< "Incompatible redo log format."
+				" The redo log was created with " << creator;
+			return DB_ERROR;
+		}
 		dberr_t err = recv_log_recover_10_3();
 		if (err != DB_SUCCESS) {
 			ib::error()
@@ -3370,7 +3388,6 @@ of first system tablespace page
 dberr_t
 recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 {
-	log_group_t*	group;
 	ulint		max_cp_field;
 	lsn_t		checkpoint_lsn;
 	bool		rescan;
@@ -3402,15 +3419,30 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 
 	err = recv_find_max_checkpoint(&max_cp_field);
 
-	if (err != DB_SUCCESS
-	    || (log_sys->log.format != 0
-		&& (log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED)
-		!= LOG_HEADER_FORMAT_CURRENT)) {
-
+	if (err != DB_SUCCESS) {
+skip_apply:
 		log_mutex_exit();
 		return(err);
 	}
 
+	switch (log_sys->log.format) {
+	case 0:
+		break;
+	case LOG_HEADER_FORMAT_10_2:
+	case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED:
+		break;
+	case LOG_HEADER_FORMAT_CURRENT:
+	case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED:
+		if (log_sys->log.subformat == 1) {
+			/* 10.2 with new crash-safe TRUNCATE */
+			break;
+		}
+		/* fall through */
+	default:
+		/* This must be a clean log from a newer version. */
+		goto skip_apply;
+	}
+
 	log_group_header_read(&log_sys->log, max_cp_field);
 
 	buf = log_sys->checkpoint_buf;
@@ -3426,13 +3458,12 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 
 	ut_ad(RECV_SCAN_SIZE <= log_sys->buf_size);
 
-	group = &log_sys->log;
 	const lsn_t	end_lsn = mach_read_from_8(
 		buf + LOG_CHECKPOINT_END_LSN);
 
 	ut_ad(recv_sys->n_addrs == 0);
 	contiguous_lsn = checkpoint_lsn;
-	switch (group->format) {
+	switch (log_sys->log.format) {
 	case 0:
 		log_mutex_exit();
 		return recv_log_format_0_recover(checkpoint_lsn,
@@ -3451,6 +3482,7 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 	}
 
 	/* Look for MLOG_CHECKPOINT. */
+	log_group_t* group = &log_sys->log;
 	recv_group_scan_log_recs(group, checkpoint_lsn, &contiguous_lsn,
 				 false);
 	/* The first scan should not have stored or applied any records. */

storage/innobase/srv/srv0start.cc

@@ -1393,6 +1393,12 @@ srv_prepare_to_delete_redo_log_files(
 	ulint	pending_io = 0;
 	ulint	count = 0;
 
+	if ((log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED)
+	    != LOG_HEADER_FORMAT_CURRENT
+	    || log_sys->log.subformat != 1) {
+		srv_log_file_size = 0;
+	}
+
 	do {
 		/* Clean the buffer pool. */
 		buf_flush_sync_all_buf_pools();
@@ -1411,7 +1417,7 @@ srv_prepare_to_delete_redo_log_files(
 			if (srv_log_file_size == 0) {
 				info << ((log_sys->log.format
 					  & ~LOG_HEADER_FORMAT_ENCRYPTED)
-					 != LOG_HEADER_FORMAT_10_3
+					 < LOG_HEADER_FORMAT_CURRENT
 					 ? "Upgrading redo log: "
 					 : "Downgrading redo log: ");
 			} else if (n_files != srv_n_log_files
@@ -2385,8 +2391,14 @@ innobase_start_or_create_for_mysql()
 			/* Leave the redo log alone. */
 		} else if (srv_log_file_size_requested == srv_log_file_size
 			   && srv_n_log_files_found == srv_n_log_files
-			   && log_sys->is_encrypted() == srv_encrypt_log) {
-			/* No need to upgrade or resize the redo log. */
+			   && log_sys->log.format
+			   == (srv_encrypt_log
+			       ? LOG_HEADER_FORMAT_CURRENT
+			       | LOG_HEADER_FORMAT_ENCRYPTED
+			       : LOG_HEADER_FORMAT_CURRENT)
+			   && log_sys->log.subformat == 1) {
+			/* No need to add or remove encryption,
+			upgrade, downgrade, or resize. */
 		} else {
 			/* Prepare to delete the old redo log files */
 			flushed_lsn = srv_prepare_to_delete_redo_log_files(i);