MDEV-14848 MariaDB 10.3 refuses InnoDB crash-upgrade from MariaDB 10.2

While the redo log format was changed in MariaDB 10.3.2 and 10.3.3
due to MDEV-12288 and MDEV-11369, it should be technically possible
to upgrade from a crashed MariaDB 10.2 instance.

On a related note, it should be possible for Mariabackup 10.3
to create a backup from a running MariaDB Server 10.2.

mlog_id_t: Put back the 10.2 specific redo log record types
MLOG_UNDO_INSERT, MLOG_UNDO_ERASE_END, MLOG_UNDO_INIT,
MLOG_UNDO_HDR_REUSE.

trx_undo_parse_add_undo_rec(): Parse or apply MLOG_UNDO_INSERT.

trx_undo_erase_page_end(): Apply MLOG_UNDO_ERASE_END.

trx_undo_parse_page_init(): Parse or apply MLOG_UNDO_INIT.

trx_undo_parse_page_header_reuse(): Parse or apply MLOG_UNDO_HDR_REUSE.

recv_log_recover_10_2(): Remove. Always parse the redo log from 10.2.

recv_find_max_checkpoint(), recv_recovery_from_checkpoint_start():
Always parse the redo log from MariaDB 10.2.

recv_parse_or_apply_log_rec_body(): Parse or apply
MLOG_UNDO_INSERT, MLOG_UNDO_ERASE_END, MLOG_UNDO_INIT.

srv_prepare_to_delete_redo_log_files(),
innobase_start_or_create_for_mysql(): Upgrade from a previous (supported)
redo log format.

mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result

@@ -40,14 +40,15 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND 1 /InnoDB: Upgrade after a crash is not supported\. The redo log was created with malicious intentions, or perhaps, and it appears corrupted\./ in mysqld.1.err
+FOUND 1 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
+FOUND 1 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # same, but with current-version header
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND 1 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
-FOUND 1 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
+FOUND 2 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
+FOUND 2 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -98,6 +99,13 @@ WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err
+# Clean 10.2 redo log
+SELECT * FROM INFORMATION_SCHEMA.ENGINES
+WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
+InnoDB	YES	Supports transactions, row-level locking, foreign keys and encryption for tables	YES	YES	YES
+FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err
 # Minimal MariaDB 10.1.21 encrypted redo log
 SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');

mysql-test/suite/innodb/r/log_corruption.result

@@ -40,14 +40,15 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND 1 /InnoDB: Upgrade after a crash is not supported\. The redo log was created with malicious intentions, or perhaps, and it appears corrupted\./ in mysqld.1.err
+FOUND 1 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
+FOUND 1 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # same, but with current-version header
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND 1 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
-FOUND 1 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
+FOUND 2 /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
+FOUND 2 /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -98,6 +99,13 @@ WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err
+# Clean 10.2 redo log
+SELECT * FROM INFORMATION_SCHEMA.ENGINES
+WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
+InnoDB	YES	Supports transactions, row-level locking, foreign keys and encryption for tables	YES	YES	YES
+FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err
 # Minimal MariaDB 10.1.21 encrypted redo log
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'

mysql-test/suite/innodb/t/log_corruption.test

@@ -204,7 +204,9 @@ EOF
 --source include/start_mysqld.inc
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
-let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported\. The redo log was created with malicious intentions, or perhaps, and it appears corrupted\.;
+let SEARCH_PATTERN=InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN=InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
 --source include/search_pattern_in_file.inc
 
 --echo # same, but with current-version header
@@ -339,8 +341,9 @@ perl;
 die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0";
 binmode OUT;
 # header block
-print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1";
-print OUT pack("x[478]N", 0x85021a0f);
+print OUT pack("Nx[5]nx[5]", 1, 0x1286);
+print OUT "ibbackup was here!!!1!";
+print OUT pack("x[470]N", 0x52b54540);
 # invalid (all-zero) checkpoint page 1 and an empty log page
 print OUT chr(0) x 1024;
 # valid checkpoint block 2
@@ -364,6 +367,28 @@ eval $check_no_innodb;
 --let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\$
 --source include/search_pattern_in_file.inc
 
+--echo # Clean 10.2 redo log
+perl;
+die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0";
+binmode OUT;
+die unless seek(OUT, 0x800, 0);
+print OUT pack("H*", "800009440022000c00000001");
+# dummy padding (MLOG_DUMMY_RECORD)
+print OUT " " x 6;
+# MLOG_CHECKPOINT record
+print OUT pack("CNN", 56, 0, 0x12860c);
+# padding (MLOG_DUMMY_RECORD) and block checksum
+print OUT " " x 481, pack("N", 0xe9b21b7b);
+close OUT or die;
+EOF
+
+--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m
+--source include/start_mysqld.inc
+eval $check_no_innodb;
+--source include/shutdown_mysqld.inc
+--let SEARCH_PATTERN= InnoDB: Upgrading redo log:
+--source include/search_pattern_in_file.inc
+
 --echo # Minimal MariaDB 10.1.21 encrypted redo log
 perl;
 die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0";

storage/innobase/include/mtr0types.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -100,6 +100,18 @@ enum mlog_id_t {
 	/** Create an index page */
 	MLOG_PAGE_CREATE = 19,
 
+	/** insert an undo log record (used in MariaDB 10.2) */
+	MLOG_UNDO_INSERT = 20,
+
+	/** erase an undo log page end (used in MariaDB 10.2) */
+	MLOG_UNDO_ERASE_END = 21,
+
+	/** initialize a page in an undo log (used in MariaDB 10.2) */
+	MLOG_UNDO_INIT = 22,
+
+	/** reuse an insert undo log header (used in MariaDB 10.2) */
+	MLOG_UNDO_HDR_REUSE = 24,
+
 	/** create an undo log header */
 	MLOG_UNDO_HDR_CREATE = 25,
 

storage/innobase/include/trx0rec.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -243,6 +243,23 @@ trx_undo_prev_version_build(
 				into this function by purge thread or not.
 				And if we read "after image" of undo log */
 
+/** Parse MLOG_UNDO_INSERT for crash-upgrade from MariaDB 10.2.
+@param[in]	ptr	log record
+@param[in]	end_ptr	end of log record buffer
+@param[in,out]	page	page or NULL
+@return	end of log record
+@retval	NULL	if the log record is incomplete */
+byte*
+trx_undo_parse_add_undo_rec(
+	const byte*	ptr,
+	const byte*	end_ptr,
+	page_t*		page);
+/** Erase the unused undo log page end.
+@param[in,out]	undo_page	undo log page
+@return whether the page contained something */
+bool
+trx_undo_erase_page_end(page_t* undo_page);
+
 /** Read from an undo log record a non-virtual column value.
 @param[in,out]	ptr		pointer to remaining part of the undo record
 @param[in,out]	field		stored field

storage/innobase/include/trx0undo.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation.
+Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -315,6 +315,32 @@ bool
 trx_undo_truncate_tablespace(
 	undo::Truncate*	undo_trunc);
 
+/** Parse MLOG_UNDO_INIT for crash-upgrade from MariaDB 10.2.
+@param[in]	ptr	log record
+@param[in]	end_ptr	end of log record buffer
+@param[in,out]	page	page or NULL
+@param[in,out]	mtr	mini-transaction
+@return	end of log record
+@retval	NULL	if the log record is incomplete */
+byte*
+trx_undo_parse_page_init(
+	const byte*	ptr,
+	const byte*	end_ptr,
+	page_t*		page,
+	mtr_t*		mtr);
+/** Parse MLOG_UNDO_HDR_REUSE for crash-upgrade from MariaDB 10.2.
+@param[in]	ptr	redo log record
+@param[in]	end_ptr	end of log buffer
+@param[in,out]	page	undo page or NULL
+@param[in,out]	mtr	mini-transaction
+@return end of log record or NULL */
+byte*
+trx_undo_parse_page_header_reuse(
+	const byte*	ptr,
+	const byte*	end_ptr,
+	page_t*		page,
+	mtr_t*		mtr);
+
 /** Parse the redo log entry of an undo log page header create.
 @param[in]	ptr	redo log record
 @param[in]	end_ptr	end of log buffer

storage/innobase/log/log0recv.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2013, 2017, MariaDB Corporation.
+Copyright (c) 2013, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -902,58 +902,6 @@ recv_log_format_0_recover(lsn_t lsn)
 	return(DB_SUCCESS);
 }
 
-/** Determine if a redo log from MySQL 5.7.9/MariaDB 10.2.2 is clean.
-@return error code
-@retval	DB_SUCCESS	if the redo log is clean
-@retval	DB_CORRUPTION	if the redo log is corrupted
-@retval DB_ERROR	if the redo log is not empty */
-static
-dberr_t
-recv_log_recover_10_2()
-{
-	log_group_t*	group = &log_sys->log;
-	const lsn_t	lsn = group->lsn;
-	const lsn_t	source_offset = log_group_calc_lsn_offset(lsn, group);
-	const ulint	page_no
-		= (ulint) (source_offset / univ_page_size.physical());
-	byte*		buf = log_sys->buf;
-
-	fil_io(IORequestLogRead, true,
-	       page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no),
-	       univ_page_size,
-	       (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1))
-			% univ_page_size.physical()),
-	       OS_FILE_LOG_BLOCK_SIZE, buf, NULL);
-
-	if (log_block_calc_checksum(buf) != log_block_get_checksum(buf)) {
-		return(DB_CORRUPTION);
-	}
-
-	if (group->is_encrypted()) {
-		log_crypt(buf, lsn, OS_FILE_LOG_BLOCK_SIZE, true);
-	}
-
-	/* On a clean shutdown, the redo log will be logically empty
-	after the checkpoint lsn. */
-
-	if (log_block_get_data_len(buf)
-	    != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) {
-		return(DB_ERROR);
-	}
-
-	/* Mark the redo log for upgrading. */
-	srv_log_file_size = 0;
-	recv_sys->parse_start_lsn = recv_sys->recovered_lsn
-		= recv_sys->scanned_lsn
-		= recv_sys->mlog_checkpoint_lsn = lsn;
-	log_sys->last_checkpoint_lsn = log_sys->next_checkpoint_lsn
-		= log_sys->lsn = log_sys->write_lsn
-		= log_sys->current_flush_lsn = log_sys->flushed_to_disk_lsn
-		= lsn;
-	log_sys->next_checkpoint_no = 0;
-	return(DB_SUCCESS);
-}
-
 /** Find the latest checkpoint in the log header.
 @param[out]	max_field	LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2
 @return error code or DB_SUCCESS */
@@ -1063,20 +1011,6 @@ recv_find_max_checkpoint(ulint* max_field)
 		return(DB_ERROR);
 	}
 
-	switch (group->format) {
-	case LOG_HEADER_FORMAT_10_2:
-	case LOG_HEADER_FORMAT_10_2 | LOG_HEADER_FORMAT_ENCRYPTED:
-		dberr_t err = recv_log_recover_10_2();
-		if (err != DB_SUCCESS) {
-			ib::error()
-				<< "Upgrade after a crash is not supported."
-				" The redo log was created with " << creator
-				<< (err == DB_ERROR
-				    ? "." : ", and it appears corrupted.");
-		}
-		return(err);
-	}
-
 	return(DB_SUCCESS);
 }
 
@@ -1394,6 +1328,25 @@ recv_parse_or_apply_log_rec_body(
 		page_parse_create(block, type == MLOG_COMP_PAGE_CREATE_RTREE,
 				  true);
 		break;
+	case MLOG_UNDO_INSERT:
+		ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG);
+		ptr = trx_undo_parse_add_undo_rec(ptr, end_ptr, page);
+		break;
+	case MLOG_UNDO_ERASE_END:
+		if (page) {
+			ut_ad(page_type == FIL_PAGE_UNDO_LOG);
+			trx_undo_erase_page_end(page);
+		}
+		break;
+	case MLOG_UNDO_INIT:
+		/* Allow anything in page_type when creating a page. */
+		ptr = trx_undo_parse_page_init(ptr, end_ptr, page, mtr);
+		break;
+	case MLOG_UNDO_HDR_REUSE:
+		ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG);
+		ptr = trx_undo_parse_page_header_reuse(ptr, end_ptr, page,
+						       mtr);
+		break;
 	case MLOG_UNDO_HDR_CREATE:
 		ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG);
 		ptr = trx_undo_parse_page_header(ptr, end_ptr, page, mtr);
@@ -3176,10 +3129,7 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 
 	err = recv_find_max_checkpoint(&max_cp_field);
 
-	if (err != DB_SUCCESS
-	    || (log_sys->log.format != LOG_HEADER_FORMAT_3_23
-		&& (log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED)
-		!= LOG_HEADER_FORMAT_CURRENT)) {
+	if (err != DB_SUCCESS) {
 
 		srv_start_lsn = recv_sys->recovered_lsn = log_sys->lsn;
 		log_mutex_exit();
@@ -3620,6 +3570,18 @@ get_mlog_string(mlog_id_t type)
 	case MLOG_PAGE_CREATE:
 		return("MLOG_PAGE_CREATE");
 
+	case MLOG_UNDO_INSERT:
+		return("MLOG_UNDO_INSERT");
+
+	case MLOG_UNDO_ERASE_END:
+		return("MLOG_UNDO_ERASE_END");
+
+	case MLOG_UNDO_INIT:
+		return("MLOG_UNDO_INIT");
+
+	case MLOG_UNDO_HDR_REUSE:
+		return("MLOG_UNDO_HDR_REUSE");
+
 	case MLOG_UNDO_HDR_CREATE:
 		return("MLOG_UNDO_HDR_CREATE");
 

storage/innobase/srv/srv0start.cc

@@ -1405,7 +1405,10 @@ srv_prepare_to_delete_redo_log_files(
 
 		{
 			ib::info	info;
-			if (srv_log_file_size == 0) {
+			if (srv_log_file_size == 0
+			    || (log_sys->log.format
+				& ~LOG_HEADER_FORMAT_ENCRYPTED)
+			    != LOG_HEADER_FORMAT_CURRENT) {
 				info << "Upgrading redo log: ";
 			} else if (n_files != srv_n_log_files
 				   || srv_log_file_size
@@ -2350,7 +2353,11 @@ innobase_start_or_create_for_mysql()
 			/* Leave the redo log alone. */
 		} else if (srv_log_file_size_requested == srv_log_file_size
 			   && srv_n_log_files_found == srv_n_log_files
-			   && log_sys->is_encrypted() == srv_encrypt_log) {
+			   && log_sys->log.format
+			   == (srv_encrypt_log
+			       ? LOG_HEADER_FORMAT_CURRENT
+			       | LOG_HEADER_FORMAT_ENCRYPTED
+			       : LOG_HEADER_FORMAT_CURRENT)) {
 			/* No need to upgrade or resize the redo log. */
 		} else {
 			/* Prepare to delete the old redo log files */