Merge 10.2 into 10.3

debian/mariadb-server-10.3.postinst

@@ -249,8 +249,8 @@ EOF
   ;;
 
   triggered)
-    if [ -x "$(command -v systemctl)" ]; then
-      systemctl daemon-reload
+    if [ -d /run/systemd/system ]; then
+      systemctl --system daemon-reload
     fi
     invoke restart
   ;;

extra/mariabackup/backup_copy.cc

@@ -1063,6 +1063,7 @@ copy_file(ds_ctxt_t *datasink,
 	ds_file_t		*dstfile = NULL;
 	datafile_cur_t		 cursor;
 	xb_fil_cur_result_t	 res;
+	DBUG_ASSERT(datasink->datasink->remove);
 	const char	*dst_path =
 		(xtrabackup_copy_back || xtrabackup_move_back)?
 		dst_file_path : trim_dotslash(dst_file_path);
@@ -1088,6 +1089,7 @@ copy_file(ds_ctxt_t *datasink,
 		if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
 			goto error;
 		}
+		DBUG_EXECUTE_IF("copy_file_error", errno=ENOSPC;goto error;);
 	}
 
 	if (res == XB_FIL_CUR_ERROR) {
@@ -1109,6 +1111,7 @@ copy_file(ds_ctxt_t *datasink,
 error:
 	datafile_close(&cursor);
 	if (dstfile != NULL) {
+		datasink->datasink->remove(dstfile->path);
 		ds_close(dstfile);
 	}
 
@@ -1153,17 +1156,18 @@ move_file(ds_ctxt_t *datasink,
 
 	if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
 		if (my_errno == EXDEV) {
-			bool ret;
-			ret = copy_file(datasink, src_file_path,
-					dst_file_path, thread_n);
+			/* Fallback to copy/unlink */
+			if(!copy_file(datasink, src_file_path,
+					dst_file_path, thread_n))
+					return false;
 			msg(thread_n,"Removing %s", src_file_path);
 			if (unlink(src_file_path) != 0) {
 				my_strerror(errbuf, sizeof(errbuf), errno);
-				msg("Error: unlink %s failed: %s",
+				msg("Warning: unlink %s failed: %s",
 					src_file_path,
 					errbuf);
 			}
-			return(ret);
+			return true;
 		}
 		my_strerror(errbuf, sizeof(errbuf), my_errno);
 		msg("Can not move file %s to %s: %s",

extra/mariabackup/datasink.h

@@ -50,9 +50,15 @@ struct datasink_struct {
 	ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
 	int (*write)(ds_file_t *file, const unsigned char *buf, size_t len);
 	int (*close)(ds_file_t *file);
+	int (*remove)(const char *path);
 	void (*deinit)(ds_ctxt_t *ctxt);
 };
 
+
+static inline int dummy_remove(const char *) {
+	return 0;
+}
+
 /* Supported datasink types */
 typedef enum {
 	DS_TYPE_STDOUT,

extra/mariabackup/ds_archive.cc

@@ -57,6 +57,7 @@ datasink_t datasink_archive = {
 	&archive_open,
 	&archive_write,
 	&archive_close,
+	&dummy_remove,
 	&archive_deinit
 };
 

extra/mariabackup/ds_buffer.cc

@@ -54,6 +54,7 @@ datasink_t datasink_buffer = {
 	&buffer_open,
 	&buffer_write,
 	&buffer_close,
+	&dummy_remove,
 	&buffer_deinit
 };
 

extra/mariabackup/ds_compress.cc

@@ -75,6 +75,7 @@ datasink_t datasink_compress = {
 	&compress_open,
 	&compress_write,
 	&compress_close,
+	&dummy_remove,
 	&compress_deinit
 };
 

extra/mariabackup/ds_local.cc

@@ -43,12 +43,18 @@ static int local_write(ds_file_t *file, const uchar *buf, size_t len);
 static int local_close(ds_file_t *file);
 static void local_deinit(ds_ctxt_t *ctxt);
 
+static int local_remove(const char *path)
+{
+	return unlink(path);
+}
+
 extern "C" {
 datasink_t datasink_local = {
 	&local_init,
 	&local_open,
 	&local_write,
 	&local_close,
+	&local_remove,
 	&local_deinit
 };
 }

extra/mariabackup/ds_stdout.cc

@@ -40,6 +40,7 @@ datasink_t datasink_stdout = {
 	&stdout_open,
 	&stdout_write,
 	&stdout_close,
+	&dummy_remove,
 	&stdout_deinit
 };
 

extra/mariabackup/ds_tmpfile.cc

@@ -51,6 +51,7 @@ datasink_t datasink_tmpfile = {
 	&tmpfile_open,
 	&tmpfile_write,
 	&tmpfile_close,
+	&dummy_remove,
 	&tmpfile_deinit
 };
 

extra/mariabackup/ds_xbstream.cc

@@ -50,6 +50,7 @@ datasink_t datasink_xbstream = {
 	&xbstream_open,
 	&xbstream_write,
 	&xbstream_close,
+	&dummy_remove,
 	&xbstream_deinit
 };
 

extra/mariabackup/innobackupex.cc

@@ -41,6 +41,7 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA
 
 #include <my_global.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <mysql.h>
 #include <my_dir.h>
@@ -206,7 +207,8 @@ enum innobackupex_options
 	OPT_STREAM,
 	OPT_TABLES_FILE,
 	OPT_THROTTLE,
-	OPT_USE_MEMORY
+	OPT_USE_MEMORY,
+	OPT_INNODB_FORCE_RECOVERY,
 };
 
 ibx_mode_t ibx_mode = IBX_MODE_BACKUP;
@@ -624,6 +626,16 @@ static struct my_option ibx_long_options[] =
 	 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
 	 1024*1024L, 0},
 
+	{"innodb-force-recovery", OPT_INNODB_FORCE_RECOVERY,
+	 "This option starts up the embedded InnoDB instance in crash "
+	 "recovery mode to ignore page corruption; should be used "
+	 "with the \"--apply-log\" option, in emergencies only. The "
+	 "default value is 0. Refer to \"innodb_force_recovery\" server "
+	 "system variable documentation for more details.",
+	 (uchar*)&xtrabackup_innodb_force_recovery,
+	 (uchar*)&xtrabackup_innodb_force_recovery,
+	 0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
 	{ 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -669,6 +681,7 @@ innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chu
 innobackupex --apply-log [--use-memory=B]\n\
              [--defaults-file=MY.CNF]\n\
              [--export] [--ibbackup=IBBACKUP-BINARY]\n\
+             [--innodb-force-recovery=1]\n\
              BACKUP-DIR\n\
 \n\
 innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
@@ -892,6 +905,12 @@ ibx_init()
 
 	opt_user = opt_ibx_user;
 	opt_password = opt_ibx_password;
+#if !defined(DONT_USE_MYSQL_PWD)
+	if (!opt_password)
+	{
+		opt_password=getenv("MYSQL_PWD");
+	}
+#endif
 	opt_host = opt_ibx_host;
 	opt_defaults_group = opt_ibx_defaults_group;
 	opt_socket = opt_ibx_socket;

extra/mariabackup/xtrabackup.cc

@@ -4,7 +4,7 @@ MariaBackup: hot backup tool for InnoDB
 Originally Created 3/3/2009 Yasufumi Kinoshita
 Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
 Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
-(c) 2017, 2020, MariaDB Corporation.
+(c) 2017, 2021, MariaDB Corporation.
 Portions written by Marko Mäkelä.
 
 This program is free software; you can redistribute it and/or modify
@@ -267,6 +267,12 @@ static char *xtrabackup_debug_sync = NULL;
 
 my_bool xtrabackup_incremental_force_scan = FALSE;
 
+/*
+ * Ignore corrupt pages (disabled by default; used
+ * by "innobackupex" as a command line argument).
+ */
+ulong xtrabackup_innodb_force_recovery = 0;
+
 /* The flushed lsn which is read from data files */
 lsn_t	flushed_lsn= 0;
 
@@ -1043,7 +1049,8 @@ enum options_xtrabackup
   OPT_ROCKSDB_DATADIR,
   OPT_BACKUP_ROCKSDB,
   OPT_XTRA_CHECK_PRIVILEGES,
-  OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION
+  OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION,
+  OPT_INNODB_FORCE_RECOVERY
 };
 
 struct my_option xb_client_options[]= {
@@ -1671,6 +1678,13 @@ struct my_option xb_server_options[] =
    &opt_check_privileges, &opt_check_privileges,
    0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
 
+  {"innodb_force_recovery", OPT_INNODB_FORCE_RECOVERY,
+   "(for --prepare): Crash recovery mode (ignores "
+   "page corruption; for emergencies only).",
+   (G_PTR*)&srv_force_recovery,
+   (G_PTR*)&srv_force_recovery,
+   0, GET_ULONG, OPT_ARG, 0, 0, SRV_FORCE_IGNORE_CORRUPT, 0, 0, 0},
+
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 
@@ -1802,24 +1816,26 @@ static int prepare_export()
       " --defaults-extra-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0 --loose-partition"
       " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
-      " --console  --skip-log-error --skip-log-bin --bootstrap  < "
+      " --console  --skip-log-error --skip-log-bin --bootstrap %s < "
       BOOTSTRAP_FILENAME IF_WIN("\"",""),
-      mariabackup_exe, 
+      mariabackup_exe,
       orig_argv1, (my_defaults_group_suffix?my_defaults_group_suffix:""),
-      xtrabackup_use_memory);
+      xtrabackup_use_memory,
+      (srv_force_recovery ? "--innodb-force-recovery=1" : ""));
   }
   else
   {
-    sprintf(cmdline,
+    snprintf(cmdline, sizeof cmdline,
      IF_WIN("\"","") "\"%s\" --mysqld"
       " --defaults-file=./backup-my.cnf --defaults-group-suffix=%s --datadir=."
       " --innodb --innodb-fast-shutdown=0 --loose-partition"
       " --innodb_purge_rseg_truncate_frequency=1 --innodb-buffer-pool-size=%llu"
-      " --console  --log-error= --skip-log-bin --bootstrap  < "
+      " --console  --log-error= --skip-log-bin --bootstrap %s < "
       BOOTSTRAP_FILENAME IF_WIN("\"",""),
       mariabackup_exe,
       (my_defaults_group_suffix?my_defaults_group_suffix:""),
-      xtrabackup_use_memory);
+      xtrabackup_use_memory,
+      (srv_force_recovery ? "--innodb-force-recovery=1" : ""));
   }
 
   msg("Prepare export : executing %s\n", cmdline);
@@ -1980,6 +1996,13 @@ xb_get_one_option(int optid,
     ADD_PRINT_PARAM_OPT(innobase_buffer_pool_filename);
     break;
 
+  case OPT_INNODB_FORCE_RECOVERY:
+
+    if (srv_force_recovery) {
+        ADD_PRINT_PARAM_OPT(srv_force_recovery);
+    }
+    break;
+
   case OPT_XTRA_TARGET_DIR:
     strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1);
     xtrabackup_target_dir= xtrabackup_real_target_dir;
@@ -2234,6 +2257,29 @@ static bool innodb_init_param()
 		srv_undo_dir = (char*) ".";
 	}
 
+	compile_time_assert(SRV_FORCE_IGNORE_CORRUPT == 1);
+
+	/*
+	 * This option can be read both from the command line, and the
+	 * defaults file. The assignment should account for both cases,
+	 * and for "--innobackupex". Since the command line argument is
+	 * parsed after the defaults file, it takes precedence.
+	 */
+	if (xtrabackup_innodb_force_recovery) {
+		srv_force_recovery = xtrabackup_innodb_force_recovery;
+	}
+
+	if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) {
+		if (!xtrabackup_prepare) {
+			msg("mariabackup: The option \"innodb_force_recovery\""
+			    " should only be used with \"%s\".",
+			    (innobackupex_mode ? "--apply-log" : "--prepare"));
+			goto error;
+		} else {
+			msg("innodb_force_recovery = %lu", srv_force_recovery);
+		}
+	}
+
 #ifdef _WIN32
 	srv_use_native_aio = TRUE;
 #endif

extra/mariabackup/xtrabackup.h

@@ -173,6 +173,8 @@ enum binlog_info_enum { BINLOG_INFO_OFF, BINLOG_INFO_ON,
 
 extern ulong opt_binlog_info;
 
+extern ulong xtrabackup_innodb_force_recovery;
+
 void xtrabackup_io_throttling(void);
 my_bool xb_write_delta_metadata(const char *filename,
 				const xb_delta_info_t *info);

mysql-test/main/userstat.result

@@ -241,6 +241,8 @@ create function f() returns int return (select 1 from performance_schema.threads
 set global userstat= 1;
 select f() from information_schema.table_statistics;
 ERROR 21000: Subquery returns more than 1 row
+select f() from information_schema.index_statistics;
+ERROR 21000: Subquery returns more than 1 row
 set global userstat= 0;
 drop function f;
 #

mysql-test/main/userstat.test

@@ -121,6 +121,8 @@ create function f() returns int return (select 1 from performance_schema.threads
 set global userstat= 1;
 --error ER_SUBQUERY_NO_1_ROW
 select f() from information_schema.table_statistics;
+--error ER_SUBQUERY_NO_1_ROW
+select f() from information_schema.index_statistics;
 set global userstat= 0;
 drop function f;
 

mysql-test/suite/galera/r/galera_virtual_blob.result

@@ -0,0 +1,19 @@
+CREATE TABLE t (f INT GENERATED ALWAYS AS (a+b)VIRTUAL,a INT,b INT,h BLOB);
+INSERT INTO t (a,b)VALUES(0,0), (0,0), (0,0), (0,0), (0,0);
+SELECT * from t;
+f	a	b	h
+0	0	0	NULL
+0	0	0	NULL
+0	0	0	NULL
+0	0	0	NULL
+0	0	0	NULL
+connection node_2;
+SELECT * from t;
+f	a	b	h
+0	0	0	NULL
+0	0	0	NULL
+0	0	0	NULL
+0	0	0	NULL
+0	0	0	NULL
+connection node_1;
+DROP TABLE t;

mysql-test/suite/galera/t/galera_virtual_blob.test

@@ -0,0 +1,10 @@
+--source include/galera_cluster.inc
+
+CREATE TABLE t (f INT GENERATED ALWAYS AS (a+b)VIRTUAL,a INT,b INT,h BLOB);
+INSERT INTO t (a,b)VALUES(0,0), (0,0), (0,0), (0,0), (0,0);
+SELECT * from t;
+
+--connection node_2
+SELECT * from t;
+--connection node_1
+DROP TABLE t;

mysql-test/suite/innodb/r/innodb_buffer_pool_fail.result

@@ -0,0 +1,7 @@
+call mtr.add_suppression("InnoDB: Cannot allocate memory for the buffer pool");
+call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc.*");
+call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
+call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
+#
+# MDEV-25019 memory allocation failures during startup cause server failure in different, confusing ways
+#

mysql-test/suite/innodb/t/innodb_buffer_pool_fail.test

@@ -0,0 +1,11 @@
+--source include/have_innodb.inc
+--source include/have_debug.inc
+call mtr.add_suppression("InnoDB: Cannot allocate memory for the buffer pool");
+call mtr.add_suppression("InnoDB: Plugin initialization aborted at srv0start.cc.*");
+call mtr.add_suppression("Plugin 'InnoDB' init function returned error.");
+call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed.");
+--echo #
+--echo # MDEV-25019 memory allocation failures during startup cause server failure in different, confusing ways
+--echo #
+let restart_parameters=--debug_dbug="+d,ib_buf_chunk_init_fails";
+--source include/restart_mysqld.inc