Merge branch '11.4' into bb-11.8-release

Author: sanja-byelkin

.github/workflows/windows-arm64.yml

@@ -54,4 +54,4 @@ jobs:
           $cidir = "$tmp/ci"
           mkdir $cidir
           fsutil file setCaseSensitiveInfo $cidir enable
-          perl bld\mysql-test\mysql-test-run.pl --force --parallel=$parallel --suite=main,innodb --vardir=$cidir/var --mysqld=--lower-case-table-names=0  --mysqld=--loose-innodb-flush-log-at-trx-commit=2
+          perl bld\mysql-test\mysql-test-run.pl --force --parallel=$parallel --suite=main,innodb --vardir=$cidir/var --mysqld=--lower-case-table-names=0  --mysqld=--loose-innodb-flush-log-at-trx-commit=2 --mysqld=--debug-no-sync

CMakeLists.txt

@@ -296,9 +296,13 @@ IF(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION
   SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_GLIBCXX_DEBUG -D_GLIBCXX_ASSERTIONS")
 ENDIF()
 
-OPTION(ENABLE_GCOV "Enable gcov (debug, Linux builds only)" OFF)
+OPTION(ENABLE_GCOV "Enable gcov (debug, macOS and Linux builds only)" OFF)
 IF (ENABLE_GCOV)
-  MY_CHECK_AND_SET_COMPILER_FLAG("-DHAVE_gcov -fprofile-arcs -ftest-coverage -lgcov" DEBUG)
+  IF (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
+    MY_CHECK_AND_SET_COMPILER_FLAG("--coverage" DEBUG)
+  ELSE()
+    MY_CHECK_AND_SET_COMPILER_FLAG("-DHAVE_gcov -fprofile-arcs -ftest-coverage -lgcov" DEBUG)
+  ENDIF()
 ENDIF()
 
 OPTION(WITHOUT_PACKED_SORT_KEYS "disable packed sort keys"  OFF)

cmake/install_macros.cmake

@@ -286,6 +286,7 @@ FUNCTION(INSTALL_RUNTIME_DEPS)
       POST_EXCLUDE_REGEXES
       ".*system32/.*\\.dll" # Windows stuff
       POST_INCLUDE_REGEXES
+      "libssl" "libcrypto" # Account for OpenSSL libraries in system32
       DIRECTORIES
       $<$<BOOL:${VCPKG_INSTALLED_DIR}>:${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin
       $<$<AND:$<CONFIG:Debug>,$<BOOL:${VCPKG_INSTALLED_DIR}>>:${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/bin>

cmake/libfmt.cmake

@@ -15,8 +15,8 @@ MACRO(BUNDLE_LIBFMT)
   ExternalProject_Add(
     libfmt
     PREFIX   "${dir}"
-    URL "https://github.com/fmtlib/fmt/releases/download/11.1.4/fmt-11.1.4.zip"
-    URL_MD5 ad6a56b15cddf4aad2a234e7cfc9e8c9
+    URL "https://github.com/fmtlib/fmt/releases/download/12.0.0/fmt-12.0.0.zip"
+    URL_MD5 9bd04e6e8c5b1733e4eefb473604219d
     INSTALL_COMMAND ""
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""

debian/mariadb-server.links

@@ -0,0 +1,6 @@
+#!/usr/bin/dh-exec
+usr/lib/${DEB_HOST_MULTIARCH}/libmariadb3/plugin/caching_sha2_password.so usr/lib/mysql/plugin/caching_sha2_password.so
+usr/lib/${DEB_HOST_MULTIARCH}/libmariadb3/plugin/client_ed25519.so usr/lib/mysql/plugin/client_ed25519.so
+usr/lib/${DEB_HOST_MULTIARCH}/libmariadb3/plugin/dialog.so usr/lib/mysql/plugin/dialog.so
+usr/lib/${DEB_HOST_MULTIARCH}/libmariadb3/plugin/mysql_clear_password.so usr/lib/mysql/plugin/mysql_clear_password.so
+usr/lib/${DEB_HOST_MULTIARCH}/libmariadb3/plugin/sha256_password.so usr/lib/mysql/plugin/sha256_password.so

extra/innochecksum.cc

@@ -51,7 +51,6 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
 #include "page0zip.h"            /* page_zip_*() */
 #include "trx0undo.h"            /* TRX_* */
 #include "fil0crypt.h"           /* fil_space_verify_crypt_checksum */
-
 #include <string.h>
 
 #ifndef PRIuMAX
@@ -74,6 +73,8 @@ static ulint extent_size;
 static ulint xdes_size;
 ulong srv_page_size;
 uint32_t srv_page_size_shift;
+static uint32_t dblwr_1;
+static uint32_t dblwr_2;
 /* Current page number (0 based). */
 uint32_t		cur_page_num;
 /* Current space. */
@@ -97,8 +98,10 @@ FILE*				log_file = NULL;
 /* Enabled for log write option. */
 static bool			is_log_enabled = false;
 static bool			skip_freed_pages;
+static uint32_t 		tablespace_flags= 0;
 static byte field_ref_zero_buf[UNIV_PAGE_SIZE_MAX];
 const byte *field_ref_zero = field_ref_zero_buf;
+constexpr uint32_t USE_FSP_FLAGS{UINT32_MAX};
 
 #ifndef _WIN32
 /* advisory lock for non-window system. */
@@ -253,12 +256,9 @@ void print_leaf_stats(
 }
 
 /** Init the page size for the tablespace.
-@param[in]	buf	buffer used to read the page */
-static void init_page_size(const byte* buf)
+@param[in]	flags	InnoDB tablespace flags */
+static void init_page_size_from_flags(const uint32_t flags)
 {
-	const unsigned	flags = mach_read_from_4(buf + FIL_PAGE_DATA
-						 + FSP_SPACE_FLAGS);
-
 	if (fil_space_t::full_crc32(flags)) {
 		const uint32_t ssize = FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(flags);
 		srv_page_size_shift = UNIV_ZIP_SIZE_SHIFT_MIN - 1 + ssize;
@@ -540,24 +540,15 @@ static bool is_page_corrupted(byte *buf, bool is_encrypted, uint32_t flags)
 	return(is_corrupted);
 }
 
-/********************************************//*
- Check if page is doublewrite buffer or not.
- @param [in] page	buffer page
-
- @retval true  if page is doublewrite buffer otherwise false.
-*/
-static
-bool
-is_page_doublewritebuffer(
-	const byte*	page)
+/** Check if page is doublewrite buffer or not.
+@retval true  if page is doublewrite buffer otherwise false. */
+static bool is_page_doublewritebuffer()
 {
-	if ((cur_page_num >= extent_size)
-		&& (cur_page_num < extent_size * 3)) {
-		/* page is doublewrite buffer. */
-		return (true);
-	}
-
-	return (false);
+  if (cur_space != 0) return false;
+  const uint32_t extent{static_cast<uint32_t>(
+    cur_page_num & ~(extent_size - 1))};
+  return cur_page_num > FSP_DICT_HDR_PAGE_NO &&
+	 extent && (extent == dblwr_1 || extent == dblwr_2);
 }
 
 /*******************************************************//*
@@ -764,7 +755,7 @@ Parse the page and collect/dump the information about page type
 @param [in] file	file for diagnosis.
 @param [in] is_encrypted  tablespace is encrypted
 */
-void
+static void
 parse_page(
 	const byte*	page,
 	byte*		xdes,
@@ -784,6 +775,12 @@ parse_page(
 	str = skip_page ? "Double_write_buffer" : "-";
 	page_no = mach_read_from_4(page + FIL_PAGE_OFFSET);
 	if (skip_freed_pages) {
+
+		/** Skip doublewrite pages when -r is enabled */
+		if (is_page_doublewritebuffer()) {
+			return;
+		}
+
 		const byte *des= xdes + XDES_ARR_OFFSET +
 			xdes_size * ((page_no & (physical_page_size - 1))
 				     / extent_size);
@@ -978,6 +975,18 @@ parse_page(
 			fprintf(file, "#::" UINT32PF "\t\t|\t\tTransaction system "
 				"page\t\t|\t%s\n", cur_page_num, str);
 		}
+
+		if (cur_space == 0 &&
+		    (mach_read_from_4(page + TRX_SYS_DOUBLEWRITE +
+			             TRX_SYS_DOUBLEWRITE_MAGIC) ==
+		     TRX_SYS_DOUBLEWRITE_MAGIC_N))  {
+			dblwr_1 = mach_read_from_4(
+					page + TRX_SYS_DOUBLEWRITE +
+					TRX_SYS_DOUBLEWRITE_BLOCK1);
+			dblwr_2 = mach_read_from_4(
+					page + TRX_SYS_DOUBLEWRITE +
+					TRX_SYS_DOUBLEWRITE_BLOCK2);
+		}
 		break;
 
 	case FIL_PAGE_TYPE_FSP_HDR:
@@ -1220,6 +1229,9 @@ static struct my_option innochecksum_options[] = {
   {"skip-freed-pages", 'r', "skip freed pages for the tablespace",
    &skip_freed_pages, &skip_freed_pages, 0, GET_BOOL, NO_ARG,
    0, 0, 0, 0, 0, 0},
+  {"tablespace-flags", 0, "InnoDB tablespace flags (default: 4294967295 "
+   "= read from page 0)", &tablespace_flags, &tablespace_flags, 0,
+   GET_UINT, REQUIRED_ARG, USE_FSP_FLAGS, 0, USE_FSP_FLAGS, 0, 0, 0},
 
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
@@ -1280,6 +1292,14 @@ innochecksum_get_one_option(
 		my_end(0);
 		exit(EXIT_SUCCESS);
 		break;
+	default:
+	  if (tablespace_flags != USE_FSP_FLAGS &&
+	      !fil_space_t::is_valid_flags(tablespace_flags, false) &&
+	      !fil_space_t::is_valid_flags(tablespace_flags, true)) {
+		fprintf(stderr, "Error: Provided --tablespace-flags "
+			"is not valid.");
+		return true;
+	  }
 	}
 
 	return(false);
@@ -1410,6 +1430,87 @@ rewrite_checksum(
 		&& !write_file(filename, fil_in, buf, flags, pos);
 }
 
+/** Read and validate page 0, then initialize tablespace flags
+and page size.
+@param	fil_in        File pointer
+@param	buf           Buffer to read page into
+@return whether the page was read successfully */
+static bool read_and_validate_page0(FILE *fil_in, byte *buf)
+{
+  /* Read the minimum page size first */
+  size_t initial_page_size= UNIV_ZIP_SIZE_MIN;
+  if (tablespace_flags != USE_FSP_FLAGS)
+  {
+    init_page_size_from_flags(tablespace_flags);
+    initial_page_size= physical_page_size;
+  }
+
+  /* Read just enough to get the tablespace flags */
+  size_t bytes= fread(buf, 1, initial_page_size, fil_in);
+
+  if (bytes != initial_page_size)
+  {
+    fprintf(stderr, "Error: Was not able to read the "
+            "minimum page size of %zu bytes. Bytes read "
+            "was %zu\n", initial_page_size, bytes);
+    return false;
+  }
+
+  /* Read space_id and page offset */
+  cur_space= mach_read_from_4(buf + FIL_PAGE_SPACE_ID);
+  cur_page_num= mach_read_from_4(buf + FIL_PAGE_OFFSET);
+
+  /* Get tablespace flags from the FSP header */
+  uint32_t flags= mach_read_from_4(buf + FSP_HEADER_OFFSET +
+                                   FSP_SPACE_FLAGS);
+
+  if (tablespace_flags != USE_FSP_FLAGS)
+  {
+    if (cur_page_num == 0 && flags != tablespace_flags)
+      fprintf(stderr, "Error: Mismatch between provided tablespace "
+              "flags (0x%x) and file flags (0x%x)\n",
+              tablespace_flags, flags);
+  }
+  else
+  {
+    if (cur_page_num)
+    {
+      fprintf(stderr, "Error: First page of the tablespace file "
+              "should be 0, but encountered page number %" PRIu32 ". "
+              "If you are checking multi file system "
+              "tablespace files, please specify the correct "
+              "tablespace flags using --tablespace-flags option.\n",
+              cur_page_num);
+      return false;
+    }
+    /* Initialize page size parameters based on flags */
+    init_page_size_from_flags(flags);
+    /* Read the rest of the page if it's larger than the minimum size */
+    if (physical_page_size > UNIV_ZIP_SIZE_MIN)
+    {
+      /* Read rest of the page 0 to determine crypt_data */
+      ulint bytes= read_file(buf, true, physical_page_size, fil_in);
+      if (bytes != physical_page_size)
+      {
+        fprintf(stderr, "Error: Was not able to read the rest of the "
+		"page of " ULINTPF " bytes. Bytes read was " ULINTPF "\n",
+		physical_page_size - UNIV_ZIP_SIZE_MIN, bytes);
+        return false;
+      }
+    }
+    tablespace_flags= flags;
+  }
+
+  if (physical_page_size < UNIV_ZIP_SIZE_MIN ||
+      physical_page_size > UNIV_PAGE_SIZE_MAX)
+  {
+    fprintf(stderr, "Error: Invalid page size " ULINTPF
+            " encountered\n", physical_page_size);
+    return false;
+  }
+  return true;
+}
+
 int main(
 	int	argc,
 	char	**argv)
@@ -1545,51 +1646,13 @@ int main(
 			}
 		}
 
-		/* Read the minimum page size. */
-		bytes = fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in);
-		partial_page_read = true;
-
-		if (bytes != UNIV_ZIP_SIZE_MIN) {
-			fprintf(stderr, "Error: Was not able to read the "
-				"minimum page size ");
-			fprintf(stderr, "of %d bytes.  Bytes read was " ULINTPF "\n",
-				UNIV_ZIP_SIZE_MIN, bytes);
-
+		/* Read and validate page 0 */
+		if (!read_and_validate_page0(fil_in, buf)) {
 			exit_status = 1;
 			goto my_exit;
 		}
 
-		/* enable variable is_system_tablespace when space_id of given
-		file is zero. Use to skip the checksum verification and rewrite
-		for doublewrite pages. */
-		cur_space = mach_read_from_4(buf + FIL_PAGE_SPACE_ID);
-		cur_page_num = mach_read_from_4(buf + FIL_PAGE_OFFSET);
-
-		/* Determine page size, zip_size and page compression
-		from fsp_flags and encryption metadata from page 0 */
-		init_page_size(buf);
-
-		uint32_t flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf);
-
-		if (physical_page_size == UNIV_ZIP_SIZE_MIN) {
-			partial_page_read = false;
-		} else {
-			/* Read rest of the page 0 to determine crypt_data */
-			bytes = read_file(buf, partial_page_read, physical_page_size, fil_in);
-			if (bytes != physical_page_size) {
-				fprintf(stderr, "Error: Was not able to read the "
-					"rest of the page ");
-				fprintf(stderr, "of " ULINTPF " bytes.  Bytes read was " ULINTPF "\n",
-					physical_page_size - UNIV_ZIP_SIZE_MIN, bytes);
-
-				exit_status = 1;
-				goto my_exit;
-			}
-			partial_page_read = false;
-		}
-
-
-		/* Now that we have full page 0 in buffer, check encryption */
+		/* Check if tablespace is encrypted */
 		bool is_encrypted = check_encryption(filename, buf);
 
 		/* Verify page 0 contents. Note that we can't allow
@@ -1600,7 +1663,8 @@ int main(
 			allow_mismatches = 0;
 
 			exit_status = verify_checksum(buf, is_encrypted,
-						      &mismatch_count, flags);
+						      &mismatch_count,
+						      tablespace_flags);
 
 			if (exit_status) {
 				fprintf(stderr, "Error: Page 0 checksum mismatch, can't continue. \n");
@@ -1611,7 +1675,8 @@ int main(
 
 		if ((exit_status = rewrite_checksum(
 					filename, fil_in, buf,
-					&pos, is_encrypted, flags))) {
+					&pos, is_encrypted,
+					tablespace_flags))) {
 			goto my_exit;
 		}
 
@@ -1807,7 +1872,7 @@ int main(
 first_non_zero:
 			if (is_system_tablespace) {
 				/* enable when page is double write buffer.*/
-				skip_page = is_page_doublewritebuffer(buf);
+				skip_page = is_page_doublewritebuffer();
 			} else {
 				skip_page = false;
 			}
@@ -1828,13 +1893,16 @@ int main(
 			    && !is_page_free(xdes, physical_page_size, cur_page_num)
 			    && (exit_status = verify_checksum(
 						buf, is_encrypted,
-						&mismatch_count, flags))) {
+						&mismatch_count,
+						tablespace_flags))) {
 				goto my_exit;
 			}
 
-			if ((exit_status = rewrite_checksum(
-						filename, fil_in, buf,
-						&pos, is_encrypted, flags))) {
+			if (!is_page_doublewritebuffer() &&
+			    (exit_status = rewrite_checksum(
+				    filename, fil_in, buf,
+				    &pos, is_encrypted,
+				    tablespace_flags))) {
 				goto my_exit;
 			}
 

extra/perror.c

@@ -353,8 +353,13 @@ int main(int argc,char *argv[])
       {
         found= 1;
         if (verbose)
+        {
+          int hundred= code / 100;
           printf("MariaDB error code %3d (%s): %s\n"
-                 "Learn more: https://mariadb.com/kb/en/e%3d/\n", code, name, msg, code);
+                 "Learn more: https://mariadb.com/docs/server/reference/"
+                 "error-codes/mariadb-error-codes-%d00-to-%d99/e%3d\n",
+                 code, name, msg, hundred, hundred, code);
+        }
         else
           puts(msg);
       }