Merge 10.2 into 10.3

extra/mariabackup/backup_copy.cc

@@ -1203,6 +1203,7 @@ copy_or_move_file(const char *src_file_path,
 
 
 
+static
 bool
 backup_files(const char *from, bool prep_mode)
 {

extra/mariabackup/xtrabackup.cc

@@ -172,7 +172,6 @@ typedef struct xb_filter_entry_struct	xb_filter_entry_t;
 lsn_t checkpoint_lsn_start;
 lsn_t checkpoint_no_start;
 static lsn_t log_copy_scanned_lsn;
-static bool log_copying;
 static bool log_copying_running;
 static bool io_watching_thread_running;
 
@@ -202,9 +201,9 @@ my_bool opt_ssl_verify_server_cert;
 /* === metadata of backup === */
 #define XTRABACKUP_METADATA_FILENAME "xtrabackup_checkpoints"
 char metadata_type[30] = ""; /*[full-backuped|log-applied|incremental]*/
-lsn_t metadata_from_lsn;
+static lsn_t metadata_from_lsn;
 lsn_t metadata_to_lsn;
-lsn_t metadata_last_lsn;
+static lsn_t metadata_last_lsn;
 
 static ds_file_t*	dst_log_file;
 
@@ -2442,25 +2441,12 @@ xtrabackup_copy_datafile(fil_node_t* node, uint thread_n)
 	return(FALSE);
 }
 
-/** How to copy a redo log segment in backup */
-enum copy_logfile {
-	/** Initial copying: copy at least one block */
-	COPY_FIRST,
-	/** Tracking while copying data files */
-	COPY_ONLINE,
-	/** Final copying: copy until the end of the log */
-	COPY_LAST
-};
-
 /** Copy redo log blocks to the data sink.
-@param[in]	copy		how to copy the log
-@param[in]	start_lsn	buffer start LSN
-@param[in]	end_lsn		buffer end LSN
-@return	last scanned LSN (equals to last copied LSN if copy=COPY_LAST)
+@param start_lsn	buffer start LSN
+@param end_lsn		buffer end LSN
+@return	last scanned LSN
 @retval	0	on failure */
-static
-lsn_t
-xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn)
+static lsn_t xtrabackup_copy_log(lsn_t start_lsn, lsn_t end_lsn)
 {
 	lsn_t	scanned_lsn	= start_lsn;
 	const byte* log_block = log_sys.buf;
@@ -2475,6 +2461,9 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn)
 		    && scanned_checkpoint - checkpoint >= 0x80000000UL) {
 			/* Garbage from a log buffer flush which was made
 			before the most recent database recovery */
+			msg("mariabackup: checkpoint wrap: "
+			    LSN_PF ",%zx,%zx\n",
+			    scanned_lsn, scanned_checkpoint, checkpoint);
 			break;
 		}
 
@@ -2495,6 +2484,8 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn)
 			   >= OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_TRL_SIZE
 			   || data_len <= LOG_BLOCK_HDR_SIZE) {
 			/* We got a garbage block (abrupt end of the log). */
+			msg("mariabackup: garbage block: " LSN_PF ",%zu\n",
+			    scanned_lsn, data_len);
 			break;
 		} else {
 			/* We got a partial block (abrupt end of the log). */
@@ -2514,7 +2505,7 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn)
 
 	log_sys.log.scanned_lsn = scanned_lsn;
 
-	end_lsn = copy == COPY_LAST
+	end_lsn = metadata_to_lsn
 		? ut_uint64_align_up(scanned_lsn, OS_FILE_LOG_BLOCK_SIZE)
 		: scanned_lsn & ~lsn_t(OS_FILE_LOG_BLOCK_SIZE - 1);
 
@@ -2534,10 +2525,8 @@ xtrabackup_copy_log(copy_logfile copy, lsn_t start_lsn, lsn_t end_lsn)
 }
 
 /** Copy redo log until the current end of the log is reached
-@param copy	how to copy the log
 @return	whether the operation failed */
-static bool
-xtrabackup_copy_logfile(copy_logfile copy)
+static bool xtrabackup_copy_logfile()
 {
 	ut_a(dst_log_file != NULL);
 	ut_ad(recv_sys != NULL);
@@ -2550,32 +2539,28 @@ xtrabackup_copy_logfile(copy_logfile copy)
 
 	start_lsn = ut_uint64_align_down(log_copy_scanned_lsn,
 					 OS_FILE_LOG_BLOCK_SIZE);
-	/* When copying the first or last part of the log, retry a few
-	times to ensure that all log up to the last checkpoint will be
-	read. */
 	do {
 		end_lsn = start_lsn + RECV_SCAN_SIZE;
 
 		xtrabackup_io_throttling();
 
 		log_mutex_enter();
-
 		lsn_t lsn= start_lsn;
-		for(int retries= 0; retries < 100; retries++) {
-			if (log_sys.log.read_log_seg(&lsn, end_lsn)) {
+		for (int retries= 0; retries < 100; retries++) {
+			if (log_sys.log.read_log_seg(&lsn, end_lsn)
+			    || lsn != start_lsn) {
 				break;
 			}
-			msg("Retrying read of a redo log block");
+			msg("Retrying read of log at LSN=" LSN_PF "\n", lsn);
 			my_sleep(1000);
 		}
 
-		start_lsn = xtrabackup_copy_log(copy, start_lsn, lsn);
+		start_lsn = (lsn == start_lsn)
+			? 0 : xtrabackup_copy_log(start_lsn, lsn);
 
 		log_mutex_exit();
 
 		if (!start_lsn) {
-			ds_close(dst_log_file);
-			dst_log_file = NULL;
 			msg("mariabackup: Error: xtrabackup_copy_logfile()"
 			    " failed.\n");
 			return(true);
@@ -2601,12 +2586,23 @@ static os_thread_ret_t DECLARE_THREAD(log_copying_thread)(void*)
 	*/
 	my_thread_init();
 
-	do {
+	for (;;) {
 		os_event_reset(log_copying_stop);
 		os_event_wait_time_low(log_copying_stop,
 				       xtrabackup_log_copy_interval * 1000ULL,
 				       0);
-	} while (log_copying && xtrabackup_copy_logfile(COPY_ONLINE));
+		if (xtrabackup_copy_logfile()) {
+			break;
+		}
+
+		log_mutex_enter();
+		bool completed = metadata_to_lsn
+			&& metadata_to_lsn < log_copy_scanned_lsn;
+		log_mutex_exit();
+		if (completed) {
+			break;
+		}
+	}
 
 	log_copying_running = false;
 	my_thread_end();
@@ -2621,7 +2617,7 @@ static os_thread_ret_t DECLARE_THREAD(io_watching_thread)(void*)
 	/* currently, for --backup only */
 	ut_a(xtrabackup_backup);
 
-	while (log_copying) {
+	while (log_copying_running && !metadata_to_lsn) {
 		os_thread_sleep(1000000); /*1 sec*/
 		io_ticket = xtrabackup_throttle;
 		os_event_set(wait_throttle);
@@ -3638,16 +3634,16 @@ xb_set_max_open_files(
 
 static void stop_backup_threads()
 {
-	log_copying = false;
-
-	if (log_copying_stop) {
+	if (log_copying_stop && log_copying_running) {
 		os_event_set(log_copying_stop);
-		msg("mariabackup: Stopping log copying thread.\n");
+		fputs("mariabackup: Stopping log copying thread", stderr);
+		fflush(stderr);
 		while (log_copying_running) {
-			msg(".");
+			putc('.', stderr);
+			fflush(stderr);
 			os_thread_sleep(200000); /*0.2 sec*/
 		}
-		msg("\n");
+		putc('\n', stderr);
 		os_event_destroy(log_copying_stop);
 	}
 
@@ -3662,10 +3658,10 @@ static void stop_backup_threads()
 
 /** Implement the core of --backup
 @return	whether the operation succeeded */
-static
-bool
-xtrabackup_backup_low()
+static bool xtrabackup_backup_low()
 {
+	ut_ad(!metadata_to_lsn);
+
 	/* read the latest checkpoint lsn */
 	{
 		ulint	max_cp_field;
@@ -3674,25 +3670,23 @@ xtrabackup_backup_low()
 
 		if (recv_find_max_checkpoint(&max_cp_field) == DB_SUCCESS
 		    && log_sys.log.format != 0) {
+			if (max_cp_field == LOG_CHECKPOINT_1) {
+				log_header_read(max_cp_field);
+			}
 			metadata_to_lsn = mach_read_from_8(
 				log_sys.checkpoint_buf + LOG_CHECKPOINT_LSN);
 			msg("mariabackup: The latest check point"
 			    " (for incremental): '" LSN_PF "'\n",
 			    metadata_to_lsn);
 		} else {
-			metadata_to_lsn = 0;
 			msg("mariabackup: Error: recv_find_max_checkpoint() failed.\n");
 		}
 		log_mutex_exit();
 	}
 
 	stop_backup_threads();
 
-	if (!dst_log_file || xtrabackup_copy_logfile(COPY_LAST)) {
-		return false;
-	}
-
-	if (ds_close(dst_log_file)) {
+	if (ds_close(dst_log_file) || !metadata_to_lsn) {
 		dst_log_file = NULL;
 		return false;
 	}
@@ -3771,6 +3765,7 @@ xtrabackup_backup_func()
 	srv_read_only_mode = TRUE;
 
 	srv_operation = SRV_OPERATION_BACKUP;
+	metadata_to_lsn = 0;
 
 	if (xb_close_files)
 		msg("mariabackup: warning: close-files specified. Use it "
@@ -3781,7 +3776,12 @@ xtrabackup_backup_func()
 	/* initialize components */
         if(innodb_init_param()) {
 fail:
+		metadata_to_lsn = log_copying_running;
 		stop_backup_threads();
+		if (dst_log_file) {
+			ds_close(dst_log_file);
+			dst_log_file = NULL;
+		}
 		if (fil_system.is_initialised()) {
 			innodb_shutdown();
 		}
@@ -3996,9 +3996,7 @@ xtrabackup_backup_func()
 		goto log_write_fail;
 	}
 
-	/* start flag */
-	log_copying = TRUE;
-
+	log_copying_running = true;
 	/* start io throttle */
 	if(xtrabackup_throttle) {
 		os_thread_id_t io_watching_thread_id;
@@ -4016,18 +4014,19 @@ xtrabackup_backup_func()
 	if (err != DB_SUCCESS) {
 		msg("mariabackup: error: xb_load_tablespaces() failed with"
 		    " error %s.\n", ut_strerr(err));
+fail_before_log_copying_thread_start:
+		log_copying_running = false;
 		goto fail;
 	}
 
 	/* copy log file by current position */
 	log_copy_scanned_lsn = checkpoint_lsn_start;
 	recv_sys->recovered_lsn = log_copy_scanned_lsn;
 
-	if (xtrabackup_copy_logfile(COPY_FIRST))
-		goto fail;
+	if (xtrabackup_copy_logfile())
+		goto fail_before_log_copying_thread_start;
 
 	log_copying_stop = os_event_create(0);
-	log_copying_running = true;
 	os_thread_create(log_copying_thread, NULL, &log_copying_thread_id);
 
 	/* FLUSH CHANGED_PAGE_BITMAPS call */

extra/mariabackup/xtrabackup.h

@@ -56,9 +56,7 @@ extern xb_page_bitmap *changed_page_bitmap;
 extern char		*xtrabackup_incremental;
 extern my_bool		xtrabackup_incremental_force_scan;
 
-extern lsn_t		metadata_from_lsn;
 extern lsn_t		metadata_to_lsn;
-extern lsn_t		metadata_last_lsn;
 
 extern xb_stream_fmt_t	xtrabackup_stream_fmt;
 extern ibool		xtrabackup_stream;

mysql-test/suite/innodb/r/innodb-blob.result

@@ -1,4 +1,4 @@
-call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles!");
+FLUSH TABLES;
 CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
 CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB;
 CREATE TABLE t3 (a INT PRIMARY KEY, b TEXT, c TEXT) ENGINE=InnoDB;

mysql-test/suite/innodb/r/innodb_defragment_small.result

@@ -1,6 +1,13 @@
 SET @innodb_defragment_orig=@@GLOBAL.innodb_defragment;
+SET @innodb_optimize_fulltext_orig=@@GLOBAL.innodb_optimize_fulltext_only;
 SET GLOBAL innodb_defragment = 1;
-CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256), KEY(a, b)) ENGINE=INNODB;
+SET GLOBAL innodb_optimize_fulltext_only = 0;
+#
+# MDEV-12198 innodb_defragment=1 crashes server on
+# OPTIMIZE TABLE when FULLTEXT index exists
+#
+CREATE TABLE t1 (a INT PRIMARY KEY, b VARCHAR(256),
+KEY(a, b), FULLTEXT KEY(b)) ENGINE=INNODB;
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
@@ -11,12 +18,15 @@ INSERT INTO t1 VALUES (400000, REPEAT('A', 256));
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
-DROP TABLE t1;
 #
-# MDEV-12198 innodb_defragment=1 crashes server on
-# OPTIMIZE TABLE when FULLTEXT index exists
+# MDEV-15824 innodb_defragment=ON trumps
+# innodb_optimize_fulltext_only=ON in OPTIMIZE TABLE
 #
-CREATE TABLE t1 (c TEXT, FULLTEXT KEY (c)) ENGINE=InnoDB;
+SET GLOBAL innodb_optimize_fulltext_only = 1;
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+SET GLOBAL innodb_defragment = 0;
 OPTIMIZE TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
@@ -27,3 +37,4 @@ Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
 DROP TABLE t1;
 SET GLOBAL innodb_defragment = @innodb_defragment_orig;
+SET GLOBAL innodb_optimize_fulltext_only = @innodb_optimize_fulltext_orig;

mysql-test/suite/innodb/t/innodb-blob.test

@@ -6,13 +6,13 @@
 # The 7000 in this test is a bit less than half the innodb_page_size.
 --source include/have_innodb_16k.inc
 
-# DEBUG_SYNC must be compiled in.
+--source include/have_debug.inc
 --source include/have_debug_sync.inc
 
 # Embedded server does not support restarting
 --source include/not_embedded.inc
 
-call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles!");
+FLUSH TABLES;
 
 CREATE TABLE t1 (a INT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
 CREATE TABLE t2 (a INT PRIMARY KEY) ENGINE=InnoDB;