Merge 10.5 into 10.6

Author: dr-m

debian/salsa-ci.yml

@@ -732,8 +732,8 @@ mariadb.org-10.2 to mariadb-10.6 upgrade:
     # prepending with --defaults-file=/etc/mysql/debian.cnf is needed in upstream 5.5–10.3
     - mysql --defaults-file=/etc/mysql/debian.cnf --skip-column-names -e "SELECT @@version, @@version_comment"
     - echo 'SHOW DATABASES;' | mysql --defaults-file=/etc/mysql/debian.cnf
-    - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT Host,User,plugin,authentication_string FROM user;" mysql
-    - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM plugin;" mysql
+    - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.global_priv; SHOW CREATE USER root@localhost; SHOW CREATE USER 'mariadb.sys'@localhost"
+    - mysql --defaults-file=/etc/mysql/debian.cnf -e "SELECT * FROM mysql.plugin; SHOW PLUGINS"
     - *test-install
     - service mysql status
     - sleep 5 # Give the mysql_upgrade a bit of time to complete before querying the server

extra/innochecksum.cc

@@ -94,6 +94,9 @@ FILE*				log_file = NULL;
 /* Enabled for log write option. */
 static bool			is_log_enabled = false;
 
+static byte field_ref_zero_buf[UNIV_PAGE_SIZE_MAX];
+const byte *field_ref_zero = field_ref_zero_buf;
+
 #ifndef _WIN32
 /* advisory lock for non-window system. */
 struct flock			lk;

extra/mariabackup/xtrabackup.cc

@@ -4479,6 +4479,14 @@ static bool xtrabackup_backup_func()
 		goto fail;
 	}
 
+
+	if (auto b = aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096)) {
+		field_ref_zero = static_cast<byte*>(
+			memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX));
+	} else {
+		goto fail;
+	}
+
         {
 	/* definition from recv_recovery_from_checkpoint_start() */
 	ulint		max_cp_field;
@@ -4493,14 +4501,17 @@ static bool xtrabackup_backup_func()
 
 	if (err != DB_SUCCESS) {
 		msg("Error: cannot read redo log header");
+unlock_and_fail:
 		mysql_mutex_unlock(&log_sys.mutex);
+free_and_fail:
+		aligned_free(const_cast<byte*>(field_ref_zero));
+		field_ref_zero = nullptr;
 		goto fail;
 	}
 
 	if (log_sys.log.format == 0) {
 		msg("Error: cannot process redo log before MariaDB 10.2.2");
-		mysql_mutex_unlock(&log_sys.mutex);
-		goto fail;
+		goto unlock_and_fail;
 	}
 
 	byte* buf = log_sys.checkpoint_buf;
@@ -4521,7 +4532,7 @@ static bool xtrabackup_backup_func()
 	xtrabackup_init_datasinks();
 
 	if (!select_history()) {
-		goto fail;
+		goto free_and_fail;
 	}
 
 	/* open the log file */
@@ -4530,12 +4541,13 @@ static bool xtrabackup_backup_func()
 	if (dst_log_file == NULL) {
 		msg("Error: failed to open the target stream for '%s'.",
 		    LOG_FILE_NAME);
-		goto fail;
+		goto free_and_fail;
 	}
 
 	/* label it */
-	byte MY_ALIGNED(OS_FILE_LOG_BLOCK_SIZE) log_hdr_buf[LOG_FILE_HDR_SIZE];
-	memset(log_hdr_buf, 0, sizeof log_hdr_buf);
+	byte* log_hdr_buf = static_cast<byte*>(
+		aligned_malloc(LOG_FILE_HDR_SIZE, OS_FILE_LOG_BLOCK_SIZE));
+	memset(log_hdr_buf, 0, LOG_FILE_HDR_SIZE);
 
 	byte *log_hdr_field = log_hdr_buf;
 	mach_write_to_4(LOG_HEADER_FORMAT + log_hdr_field, log_sys.log.format);
@@ -4564,11 +4576,13 @@ static bool xtrabackup_backup_func()
 			log_block_calc_checksum_crc32(log_hdr_field));
 
 	/* Write log header*/
-	if (ds_write(dst_log_file, log_hdr_buf, sizeof(log_hdr_buf))) {
+	if (ds_write(dst_log_file, log_hdr_buf, LOG_FILE_HDR_SIZE)) {
 		msg("error: write to logfile failed");
-		goto fail;
+		aligned_free(log_hdr_buf);
+		goto free_and_fail;
 	}
 
+	aligned_free(log_hdr_buf);
 	log_copying_running = true;
 	/* start io throttle */
 	if(xtrabackup_throttle) {
@@ -4586,7 +4600,7 @@ static bool xtrabackup_backup_func()
 		    " error %s.", ut_strerr(err));
 fail_before_log_copying_thread_start:
 		log_copying_running = false;
-		goto fail;
+		goto free_and_fail;
 	}
 
 	/* copy log file by current position */
@@ -4609,7 +4623,7 @@ static bool xtrabackup_backup_func()
 
 	/* FLUSH CHANGED_PAGE_BITMAPS call */
 	if (!flush_changed_page_bitmaps()) {
-		goto fail;
+		goto free_and_fail;
 	}
 
 	ut_a(xtrabackup_parallel > 0);
@@ -4677,6 +4691,9 @@ static bool xtrabackup_backup_func()
 	if (opt_log_innodb_page_corruption)
 		ok = corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE);
 
+	aligned_free(const_cast<byte*>(field_ref_zero));
+	field_ref_zero = nullptr;
+
 	if (!ok) {
 		goto fail;
 	}

mysql-test/main/crash_commit_before-master.opt

@@ -1,3 +1,3 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file
 --default-storage-engine=MyISAM
 --loose-skip-innodb-file-per-table

mysql-test/main/myisam_crash_before_flush_keys-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/binlog/t/binlog_index-master.opt

@@ -1 +1 @@
---skip-stack-trace --log-warnings=0 --log-bin=master-bin --log-bin-index=master-bin
+--loose-skip-stack-trace --log-warnings=0 --log-bin=master-bin --log-bin-index=master-bin

mysql-test/suite/binlog/t/binlog_mdev342-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/binlog/t/binlog_xa_recover.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --loose-debug-dbug=+d,xa_recover_expect_master_bin_000004
+--loose-skip-stack-trace --skip-core-file --loose-debug-dbug=+d,xa_recover_expect_master_bin_000004

mysql-test/suite/innodb/t/group_commit_binlog_pos-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/innodb/t/group_commit_binlog_pos_no_optimize_thread-master.opt

@@ -1 +1 @@
---binlog-optimize-thread-scheduling=0 --skip-stack-trace --skip-core-file
+--binlog-optimize-thread-scheduling=0 --loose-skip-stack-trace --skip-core-file

mysql-test/suite/innodb/t/group_commit_crash-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread-master.opt

@@ -1 +1 @@
---binlog-optimize-thread-scheduling=0 --skip-stack-trace --skip-core-file
+--binlog-optimize-thread-scheduling=0 --loose-skip-stack-trace --skip-core-file

mysql-test/suite/innodb/t/innodb_bug14147491-master.opt

@@ -1,4 +1,4 @@
 --innodb_file_per_table=1
---skip-stack-trace
+--loose-skip-stack-trace
 --skip-core-file
 --loose-innodb_buffer_pool_load_at_startup=OFF

mysql-test/suite/maria/bulk_insert_crash.opt

@@ -1,2 +1,2 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file
 --default-storage-engine=Aria

mysql-test/suite/maria/maria-gis-recovery.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp
+--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp

mysql-test/suite/maria/maria-recovery-big-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --max_allowed_packet=32000000
+--loose-skip-stack-trace --skip-core-file --max_allowed_packet=32000000

mysql-test/suite/maria/maria-recovery-bitmap-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/maria/maria-recovery-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp
+--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp

mysql-test/suite/maria/maria-recovery-rtree-ft-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/maria/maria-recovery2-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp --myisam-recover-options=
+--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp --myisam-recover-options=

mysql-test/suite/maria/maria-recovery3-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp
+--loose-skip-stack-trace --skip-core-file --loose-aria-log-dir-path=$MYSQLTEST_VARDIR/tmp

mysql-test/suite/parts/t/debug_innodb_crash-master.opt

@@ -1 +1 @@
---loose-innodb-file-per-table=1 --skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M
+--loose-innodb-file-per-table=1 --loose-skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M

mysql-test/suite/parts/t/debug_myisam_crash-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --myisam-recover-options=off
+--loose-skip-stack-trace --skip-core-file --myisam-recover-options=off

mysql-test/suite/parts/t/partition_debug_innodb-master.opt

@@ -1 +1 @@
---loose-innodb-file-per-table=1 --skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M
+--loose-innodb-file-per-table=1 --loose-skip-stack-trace --skip-core-file --loose-innodb-buffer-pool-size=32M

mysql-test/suite/rpl/t/rpl_gtid_crash-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysql-test/suite/rpl/t/rpl_gtid_crash_myisam-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file --skip-innodb
+--loose-skip-stack-trace --skip-core-file --skip-innodb

mysql-test/suite/rpl/t/rpl_parallel_temptable-master.opt

@@ -1 +1 @@
---skip-stack-trace --skip-core-file
+--loose-skip-stack-trace --skip-core-file

mysys/crc32/crc32_arm64.c

@@ -2,13 +2,13 @@
 #include <string.h>
 #include <stdint.h>
 
+static int pmull_supported;
+
 #if defined(HAVE_ARMV8_CRC)
 
 #if defined(__APPLE__)
 #include <sys/sysctl.h>
 
-static int pmull_supported;
-
 int crc32_aarch64_available(void)
 {
   int ret;
@@ -48,8 +48,6 @@ static unsigned long getauxval(unsigned int key)
 # define HWCAP_PMULL (1 << 4)
 #endif
 
-static int pmull_supported;
-
 /* ARM made crc32 default from ARMv8.1 but optional in ARMv8A
  * Runtime check API.
  */

storage/innobase/buf/buf0buf.cc

@@ -318,6 +318,12 @@ constexpr ulint	BUF_PAGE_READ_MAX_RETRIES= 100;
 read-ahead buffer.  (Divide buf_pool size by this amount) */
 constexpr uint32_t BUF_READ_AHEAD_PORTION= 32;
 
+/** A 64KiB buffer of NUL bytes, for use in assertions and checks,
+and dummy default values of instantly dropped columns.
+Initially, BLOB field references are set to NUL bytes, in
+dtuple_convert_big_rec(). */
+const byte *field_ref_zero;
+
 /** The InnoDB buffer pool */
 buf_pool_t buf_pool;
 buf_pool_t::chunk_t::map *buf_pool_t::chunk_t::map_reg;
@@ -571,7 +577,7 @@ static void buf_page_check_lsn(bool check_lsn, const byte* read_buf)
 @return whether the buffer is all zeroes */
 bool buf_is_zeroes(span<const byte> buf)
 {
-  ut_ad(buf.size() <= sizeof field_ref_zero);
+  ut_ad(buf.size() <= UNIV_PAGE_SIZE_MAX);
   return memcmp(buf.data(), field_ref_zero, buf.size()) == 0;
 }
 
@@ -1151,11 +1157,17 @@ bool buf_pool_t::create()
   ut_ad(srv_buf_pool_size % srv_buf_pool_chunk_unit == 0);
   ut_ad(!is_initialised());
   ut_ad(srv_buf_pool_size > 0);
+  ut_ad(!resizing);
+  ut_ad(!chunks_old);
+  ut_ad(!field_ref_zero);
 
   NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE;
 
-  ut_ad(!resizing);
-  ut_ad(!chunks_old);
+  if (auto b= aligned_malloc(UNIV_PAGE_SIZE_MAX, 4096))
+    field_ref_zero= static_cast<const byte*>
+      (memset_aligned<4096>(b, 0, UNIV_PAGE_SIZE_MAX));
+  else
+    return true;
 
   chunk_t::map_reg= UT_NEW_NOKEY(chunk_t::map());
 
@@ -1186,6 +1198,8 @@ bool buf_pool_t::create()
       chunks= nullptr;
       UT_DELETE(chunk_t::map_reg);
       chunk_t::map_reg= nullptr;
+      aligned_free(const_cast<byte*>(field_ref_zero));
+      field_ref_zero= nullptr;
       ut_ad(!is_initialised());
       return true;
     }
@@ -1301,6 +1315,8 @@ void buf_pool_t::close()
   io_buf.close();
   UT_DELETE(chunk_t::map_reg);
   chunk_t::map_reg= chunk_t::map_ref= nullptr;
+  aligned_free(const_cast<byte*>(field_ref_zero));
+  field_ref_zero= nullptr;
 }
 
 /** Try to reallocate a control block.
@@ -1326,6 +1342,7 @@ inline bool buf_pool_t::realloc(buf_block_t *block)
 	if (block->page.can_relocate()) {
 		memcpy_aligned<OS_FILE_LOG_BLOCK_SIZE>(
 			new_block->frame, block->frame, srv_page_size);
+		mysql_mutex_lock(&buf_pool.flush_list_mutex);
 		new (&new_block->page) buf_page_t(block->page);
 
 		/* relocate LRU list */
@@ -1385,6 +1402,7 @@ inline bool buf_pool_t::realloc(buf_block_t *block)
 			buf_flush_relocate_on_flush_list(&block->page,
 							 &new_block->page);
 		}
+		mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 		block->page.set_corrupt_id();
 
 		/* set other flags of buf_block_t */
@@ -2780,12 +2798,14 @@ buf_page_get_low(
 		/* Note: this is the uncompressed block and it is not
 		accessible by other threads yet because it is not in
 		any list or hash table */
+		mysql_mutex_lock(&buf_pool.flush_list_mutex);
 		buf_relocate(bpage, &block->page);
 
 		/* Set after buf_relocate(). */
 		block->page.set_buf_fix_count(1);
 
 		buf_flush_relocate_on_flush_list(bpage, &block->page);
+		mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 
 		/* Buffer-fix, I/O-fix, and X-latch the block
 		for the duration of the decompression.
@@ -3243,8 +3263,10 @@ static buf_block_t* buf_page_create_low(page_id_t page_id, ulint zip_size,
       }
 
       free_block->lock.x_lock();
+      mysql_mutex_lock(&buf_pool.flush_list_mutex);
       buf_relocate(&block->page, &free_block->page);
       buf_flush_relocate_on_flush_list(&block->page, &free_block->page);
+      mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 
       free_block->page.set_state(BUF_BLOCK_FILE_PAGE);
       buf_unzip_LRU_add_block(free_block, FALSE);

storage/innobase/buf/buf0flu.cc

@@ -289,43 +289,29 @@ buf_flush_relocate_on_flush_list(
 {
 	buf_page_t*	prev;
 
-	mysql_mutex_assert_owner(&buf_pool.mutex);
+	mysql_mutex_assert_owner(&buf_pool.flush_list_mutex);
 	ut_ad(!fsp_is_system_temporary(bpage->id().space()));
 
-	const lsn_t lsn = bpage->oldest_modification_acquire();
+	const lsn_t lsn = bpage->oldest_modification();
 
 	if (!lsn) {
 		return;
 	}
 
 	ut_ad(lsn == 1 || lsn > 2);
-
-	mysql_mutex_lock(&buf_pool.flush_list_mutex);
-
-	/* FIXME: Can we avoid holding buf_pool.mutex here? */
 	ut_ad(dpage->oldest_modification() == lsn);
 
-	if (ut_d(const lsn_t o_lsn =) bpage->oldest_modification()) {
-		ut_ad(o_lsn == lsn);
+	/* Important that we adjust the hazard pointer before removing
+	the bpage from the flush list. */
+	buf_pool.flush_hp.adjust(bpage);
 
-		/* Important that we adjust the hazard pointer before removing
-		the bpage from the flush list. */
-		buf_pool.flush_hp.adjust(bpage);
+	prev = UT_LIST_GET_PREV(list, bpage);
+	UT_LIST_REMOVE(buf_pool.flush_list, bpage);
 
-		prev = UT_LIST_GET_PREV(list, bpage);
-		UT_LIST_REMOVE(buf_pool.flush_list, bpage);
-
-		bpage->clear_oldest_modification();
-	} else {
-		/* bpage was removed from buf_pool.flush_list
-		since we last checked, and before we acquired
-		buf_pool.flush_list_mutex. */
-		goto was_clean;
-	}
+	bpage->clear_oldest_modification();
 
 	if (lsn == 1) {
 		buf_pool.stat.flush_list_bytes -= dpage->physical_size();
-was_clean:
 		dpage->list.prev = nullptr;
 		dpage->list.next = nullptr;
 		dpage->clear_oldest_modification();
@@ -337,7 +323,6 @@ buf_flush_relocate_on_flush_list(
 	}
 
 	ut_d(buf_flush_validate_low());
-	mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 }
 
 /** Complete write of a file page from buf_pool.