MDEV-17958 Make bug-endian innodb_checksum_algorithm=crc32 optional

In MySQL 5.7, it was noticed that files are not portable between
big-endian and little-endian processor architectures
(such as SPARC and x86), because the original implementation of
innodb_checksum_algorithm=crc32 was not byte order agnostic.

A byte order agnostic implementation of innodb_checksum_algorithm=crc32
was only added to MySQL 5.7, not backported to 5.6. Consequently,
MariaDB Server versions 10.0 and 10.1 only contain the CRC-32C
implementation that works incorrectly on big-endian architectures,
and MariaDB Server 10.2.2 got the byte-order agnostic CRC-32C
implementation from MySQL 5.7.

MySQL 5.7 introduced a "legacy crc32" variant that is functionally
equivalent to the big-endian version of the original crc32 implementation.
Thanks to this variant, old data files can be transferred from big-endian
systems to newer versions.

Introducing new variants of checksum algorithms (without introducing
new names for them, or something on the pages themselves to identify
the algorithm) generally is a bad idea, because each checksum algorithm
is like a lottery ticket. The more algorithms you try, the more likely
it will be for the checksum to match on a corrupted page.

So, essentially MySQL 5.7 weakened innodb_checksum_algorithm=crc32,
and MariaDB 10.2.2 inherited this weakening.

We introduce a build option that together with MDEV-17957
makes innodb_checksum_algorithm=strict_crc32 strict again
by only allowing one variant of the checksum to match.

WITH_INNODB_BUG_ENDIAN_CRC32: A new cmake option for enabling the
bug-compatible "legacy crc32" checksum. This is only enabled on
big-endian systems by default, to facilitate an upgrade from
MariaDB 10.0 or 10.1. Checked by #ifdef INNODB_BUG_ENDIAN_CRC32.

ut_crc32_byte_by_byte: Remove (unused function).

legacy_big_endian_checksum: Remove. This variable seems to have
unnecessarily complicated the logic. When the weakening is enabled,
we must always fall back to the buggy checksum.

buf_page_check_crc32(): A helper function to compute one or
two CRC-32C variants.

Author: dr-m

storage/innobase/buf/buf0buf.cc

@@ -756,17 +756,14 @@ buf_page_is_zeroes(
 @param[in]	read_buf		database page
 @param[in]	checksum_field1		new checksum field
 @param[in]	checksum_field2		old checksum field
-@param[in]	use_legacy_big_endian   use legacy big endian algorithm
 @return true if the page is in crc32 checksum format. */
 bool
 buf_page_is_checksum_valid_crc32(
 	const byte*			read_buf,
 	ulint				checksum_field1,
-	ulint				checksum_field2,
-	bool				use_legacy_big_endian)
+	ulint				checksum_field2)
 {
-	const uint32_t	crc32 = buf_calc_page_crc32(read_buf,
-						    use_legacy_big_endian);
+	const uint32_t	crc32 = buf_calc_page_crc32(read_buf);
 
 #ifdef UNIV_INNOCHECKSUM
 	if (log_file
@@ -783,17 +780,11 @@ buf_page_is_checksum_valid_crc32(
 		return false;
 	}
 
-	if (checksum_field1 == crc32) {
-		return(true);
-	} else {
-		const uint32_t	crc32_legacy = buf_calc_page_crc32(read_buf, true);
-
-		if (checksum_field1 == crc32_legacy) {
-			return(true);
-		}
-	}
-
-	return(false);
+	return checksum_field1 == crc32
+#ifdef INNODB_BUG_ENDIAN_CRC32
+		|| checksum_field1 == buf_calc_page_crc32(read_buf, true)
+#endif
+		;
 }
 
 /** Checks if the page is in innodb checksum format.
@@ -922,6 +913,29 @@ buf_page_is_checksum_valid_none(
 	       && checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
 }
 
+#ifdef INNODB_BUG_ENDIAN_CRC32
+/** Validate the CRC-32C checksum of a page.
+@param[in]	page		buffer page (srv_page_size bytes)
+@param[in]	checksum	CRC-32C checksum stored on page
+@return	computed checksum */
+static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum)
+{
+	uint32_t crc32 = buf_calc_page_crc32(page);
+
+	if (checksum != crc32) {
+		crc32 = buf_calc_page_crc32(page, true);
+	}
+
+	return crc32;
+}
+#else /* INNODB_BUG_ENDIAN_CRC32 */
+/** Validate the CRC-32C checksum of a page.
+@param[in]	page		buffer page (srv_page_size bytes)
+@param[in]	checksum	CRC-32C checksum stored on page
+@return	computed checksum */
+# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page)
+#endif /* INNODB_BUG_ENDIAN_CRC32 */
+
 /** Check if a page is corrupt.
 @param[in]	check_lsn	whether the LSN should be checked
 @param[in]	read_buf	database page
@@ -1037,26 +1051,20 @@ buf_page_is_corrupted(
 	    && *reinterpret_cast<const ib_uint64_t*>(
 		    read_buf + FIL_PAGE_LSN) == 0) {
 
-		ulint i;
-
 		/* make sure that the page is really empty */
-		for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) {
+		for (ulint i = 0; i < page_size.logical(); i++) {
 			if (read_buf[i] != 0) {
 				return(true);
 			}
 		}
 #ifdef UNIV_INNOCHECKSUM
-		if (i >= page_size.logical()) {
-			if (log_file) {
-				fprintf(log_file, "Page::%llu"
-					" is empty and uncorrupted\n",
-					cur_page_num);
-			}
-			return(false);
+		if (log_file) {
+			fprintf(log_file, "Page::%llu"
+				" is empty and uncorrupted\n",
+				cur_page_num);
 		}
-#else
-		return(i < page_size.logical());
 #endif /* UNIV_INNOCHECKSUM */
+		return(false);
 	}
 
 	const srv_checksum_algorithm_t	curr_algo =
@@ -1065,10 +1073,7 @@ buf_page_is_corrupted(
 	switch (curr_algo) {
 	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
 		return !buf_page_is_checksum_valid_crc32(
-			read_buf, checksum_field1, checksum_field2, false)
-			&& !buf_page_is_checksum_valid_crc32(
-				read_buf, checksum_field1, checksum_field2,
-				true);
+			read_buf, checksum_field1, checksum_field2);
 	case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
 		return !buf_page_is_checksum_valid_innodb(
 			read_buf, checksum_field1, checksum_field2);
@@ -1111,19 +1116,10 @@ buf_page_is_corrupted(
 
 			if (srv_checksum_algorithm
 			    == SRV_CHECKSUM_ALGORITHM_CRC32) {
-
-				crc32 = buf_calc_page_crc32(
-					read_buf, legacy_big_endian_checksum);
+				crc32 = buf_page_check_crc32(read_buf,
+							     checksum_field2);
 				crc32_inited = true;
 
-				if (!legacy_big_endian_checksum
-				    && checksum_field2 != crc32) {
-					crc32 = buf_calc_page_crc32(read_buf,
-								    true);
-					legacy_big_endian_checksum =
-						checksum_field2 == crc32;
-				}
-
 				if (checksum_field2 != crc32
 				    && checksum_field2
 				       != buf_calc_page_old_checksum(read_buf)) {
@@ -1135,19 +1131,10 @@ buf_page_is_corrupted(
 
 				if (checksum_field2
 				    != buf_calc_page_old_checksum(read_buf)) {
-					crc32 = buf_calc_page_crc32(
-						read_buf,
-						legacy_big_endian_checksum);
+					crc32 = buf_page_check_crc32(
+						read_buf, checksum_field2);
 					crc32_inited = true;
 
-					if (!legacy_big_endian_checksum
-					    && checksum_field2 != crc32) {
-						crc32 = buf_calc_page_crc32(
-							read_buf, true);
-						legacy_big_endian_checksum =
-							checksum_field2 == crc32;
-					}
-
 					if (checksum_field2 != crc32) {
 						return true;
 					}
@@ -1161,18 +1148,9 @@ buf_page_is_corrupted(
 			   == SRV_CHECKSUM_ALGORITHM_CRC32) {
 
 			if (!crc32_inited) {
-				crc32 = buf_calc_page_crc32(
-					read_buf,
-					legacy_big_endian_checksum);
+				crc32 = buf_page_check_crc32(
+					read_buf, checksum_field2);
 				crc32_inited = true;
-
-				if (!legacy_big_endian_checksum
-				    && checksum_field2 != crc32) {
-					crc32 = buf_calc_page_crc32(
-						read_buf, true);
-					legacy_big_endian_checksum =
-						checksum_field2 == crc32;
-				}
 			}
 
 			if (checksum_field1 != crc32
@@ -1188,18 +1166,9 @@ buf_page_is_corrupted(
 			    != buf_calc_page_new_checksum(read_buf)) {
 
 				if (!crc32_inited) {
-					crc32 = buf_calc_page_crc32(
-						read_buf,
-						legacy_big_endian_checksum);
+					crc32 = buf_page_check_crc32(
+						read_buf, checksum_field2);
 					crc32_inited = true;
-
-					if (!legacy_big_endian_checksum
-					    && checksum_field2 != crc32) {
-						crc32 = buf_calc_page_crc32(
-							read_buf, true);
-						legacy_big_endian_checksum =
-							checksum_field2 == crc32;
-					}
 				}
 
 				if (checksum_field1 != crc32) {
@@ -1255,10 +1224,12 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< page_zip_calc_checksum(
 				read_buf, page_size.physical(),
 				SRV_CHECKSUM_ALGORITHM_CRC32)
+#ifdef INNODB_BUG_ENDIAN_CRC32
 			<< "/"
 			<< page_zip_calc_checksum(
 				read_buf, page_size.physical(),
 				SRV_CHECKSUM_ALGORITHM_CRC32, true)
+#endif
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB)
@@ -1284,9 +1255,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 
 	} else {
 		const uint32_t	crc32 = buf_calc_page_crc32(read_buf);
-
+#ifdef INNODB_BUG_ENDIAN_CRC32
 		const uint32_t	crc32_legacy = buf_calc_page_crc32(read_buf,
 								   true);
+#endif /* INNODB_BUG_ENDIAN_CRC32 */
 		ulint page_type = fil_page_get_type(read_buf);
 
 		ib::info() << "Uncompressed page, stored checksum in field1 "
@@ -1295,7 +1267,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< ", calculated checksums for field1: "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_CRC32) << " "
-			<< crc32 << "/" << crc32_legacy
+			<< crc32
+#ifdef INNODB_BUG_ENDIAN_CRC32
+			<< "/" << crc32_legacy
+#endif
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@@ -1312,7 +1287,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< ", calculated checksums for field2: "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_CRC32) << " "
-			<< crc32 << "/" << crc32_legacy
+			<< crc32
+#ifdef INNODB_BUG_ENDIAN_CRC32
+			<< "/" << crc32_legacy
+#endif
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@@ -3950,10 +3928,12 @@ buf_zip_decompress(
 			<< ", crc32: "
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_CRC32)
+#ifdef INNODB_BUG_ENDIAN_CRC32
 			<< "/"
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_CRC32,
 				true)
+#endif
 			<< " innodb: "
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_INNODB)

storage/innobase/buf/buf0checksum.cc

@@ -39,44 +39,56 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
   'long unsigned int*' in initialization */
 ulong	srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
 
-/** set if we have found pages matching legacy big endian checksum */
-bool	legacy_big_endian_checksum = false;
-/** Calculates the CRC32 checksum of a page. The value is stored to the page
+#ifdef INNODB_BUG_ENDIAN_CRC32
+/** Calculate the CRC32 checksum of a page. The value is stored to the page
 when it is written to a file and also checked for a match when reading from
-the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian
-variants. Note that we must be careful to calculate the same value on 32-bit
-and 64-bit architectures.
-@param[in]	page			buffer page (UNIV_PAGE_SIZE bytes)
-@param[in]	use_legacy_big_endian	if true then use big endian
-byteorder when converting byte strings to integers
-@return checksum */
-uint32_t
-buf_calc_page_crc32(
-	const byte*	page,
-	bool		use_legacy_big_endian /* = false */)
+the file. Note that we must be careful to calculate the same value on all
+architectures.
+@param[in]	page		buffer page (srv_page_size bytes)
+@param[in]	bug_endian	whether to use big endian byteorder
+when converting byte strings to integers, for bug-compatibility with
+big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
+@return	CRC-32C */
+uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian)
 {
-	/* Since the field FIL_PAGE_FILE_FLUSH_LSN, and in versions <= 4.1.x
-	FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, are written outside the buffer pool
-	to the first pages of data files, we have to skip them in the page
-	checksum calculation.
-	We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the
-	checksum is stored, and also the last 8 bytes of page because
-	there we store the old formula checksum. */
-
-	ut_crc32_func_t	crc32_func = use_legacy_big_endian
-		? ut_crc32_legacy_big_endian
-		: ut_crc32;
-
-	const uint32_t	c1 = crc32_func(
-		page + FIL_PAGE_OFFSET,
-		FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - FIL_PAGE_OFFSET);
-
-	const uint32_t	c2 = crc32_func(
-		page + FIL_PAGE_DATA,
-		UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_END_LSN_OLD_CHKSUM);
-
-	return(c1 ^ c2);
+	return bug_endian
+		? ut_crc32_legacy_big_endian(
+			page + FIL_PAGE_OFFSET,
+			FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+			- FIL_PAGE_OFFSET)
+		^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA,
+					     srv_page_size
+					     - (FIL_PAGE_DATA
+						+ FIL_PAGE_END_LSN_OLD_CHKSUM))
+		: ut_crc32(page + FIL_PAGE_OFFSET,
+			   FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+			   - FIL_PAGE_OFFSET)
+		^ ut_crc32(page + FIL_PAGE_DATA,
+			   srv_page_size
+			   - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
+}
+#else
+/** Calculate the CRC32 checksum of a page. The value is stored to the page
+when it is written to a file and also checked for a match when reading from
+the file. Note that we must be careful to calculate the same value on all
+architectures.
+@param[in]	page			buffer page (srv_page_size bytes)
+@return	CRC-32C */
+uint32_t buf_calc_page_crc32(const byte* page)
+{
+	/* Note: innodb_checksum_algorithm=crc32 could and should have
+	included the entire page in the checksum, and CRC-32 values
+	should be combined with the CRC-32 function, not with
+	exclusive OR. We stick to the current algorithm in order to
+	remain compatible with old data files. */
+	return ut_crc32(page + FIL_PAGE_OFFSET,
+			FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
+			- FIL_PAGE_OFFSET)
+		^ ut_crc32(page + FIL_PAGE_DATA,
+			   srv_page_size
+			   - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
 }
+#endif
 
 /** Calculate a checksum which is stored to the page when it is written
 to a file. Note that we must be careful to calculate the same value on

storage/innobase/fil/fil0crypt.cc

@@ -2636,10 +2636,8 @@ fil_space_verify_crypt_checksum(
 					srv_checksum_algorithm);
 		switch (algorithm) {
 		case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-			/* We never supported upgrade from the "legacy crc32"
-			on big endian systems from MariaDB 10.1 to later. */
 			valid = buf_page_is_checksum_valid_crc32(
-				page, checksum1, checksum2, false);
+				page, checksum1, checksum2);
 			break;
 		case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
 			valid = buf_page_is_checksum_valid_innodb(
@@ -2649,13 +2647,11 @@ fil_space_verify_crypt_checksum(
 		case SRV_CHECKSUM_ALGORITHM_CRC32:
 		case SRV_CHECKSUM_ALGORITHM_INNODB:
 		case SRV_CHECKSUM_ALGORITHM_NONE:
-			/* We never supported upgrade from the "legacy crc32"
-			on big endian systems from MariaDB 10.1 to later.
-			We also never supported
+			/* never supported
 			innodb_checksum_algorithm=none or strict_none
 			for encrypted pages. */
 			valid = buf_page_is_checksum_valid_crc32(
-				page, checksum1, checksum2, false)
+				page, checksum1, checksum2)
 				|| buf_page_is_checksum_valid_innodb(
 					page, checksum1, checksum2);
 			break;
@@ -2684,8 +2680,11 @@ fil_space_verify_crypt_checksum(
 		ib::info()
 			<< "If unencrypted: stored checksum [" << checksum1
 			<< ":" << checksum2 << "] calculated crc32 ["
-			<< buf_calc_page_crc32(page, false) << ":"
-			<< buf_calc_page_crc32(page, true) << "] innodb ["
+			<< buf_calc_page_crc32(page)
+# ifdef INNODB_BUG_ENDIAN_CRC32
+			<< ":" << buf_calc_page_crc32(page, true)
+# endif /* INNODB_BUG_ENDIAN_CRC32 */
+			<< "] innodb ["
 			<< buf_calc_page_old_checksum(page) << ":"
 			<< buf_calc_page_new_checksum(page) << "] LSN "
 			<< mach_read_from_4(page + FIL_PAGE_LSN);