Merge 10.5 into 10.6

Author: dr-m

extra/mariabackup/backup_copy.cc

@@ -868,21 +868,14 @@ datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
 	return(true);
 }
 
-
-static
-bool
-backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
+bool backup_file_print_buf(const char *filename, const char *buf, int buf_len)
 {
 	ds_file_t	*dstfile	= NULL;
 	MY_STAT		 stat;			/* unused for now */
-	char		*buf		= 0;
-	int		 buf_len;
 	const char	*action;
 
 	memset(&stat, 0, sizeof(stat));
 
-	buf_len = vasprintf(&buf, fmt, ap);
-
 	stat.st_size = buf_len;
 	stat.st_mtime = my_time(0);
 
@@ -906,7 +899,6 @@ backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
 
 	/* close */
 	msg("        ...done");
-	free(buf);
 
 	if (ds_close(dstfile)) {
 		goto error_close;
@@ -915,16 +907,28 @@ backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
 	return(true);
 
 error:
-	free(buf);
 	if (dstfile != NULL) {
 		ds_close(dstfile);
 	}
 
 error_close:
 	msg("Error: backup file failed.");
 	return(false); /*ERROR*/
-}
 
+	return true;
+};
+
+static
+bool
+backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
+{
+	char		*buf		= 0;
+	int		 buf_len;
+	buf_len = vasprintf(&buf, fmt, ap);
+	bool result = backup_file_print_buf(filename, buf, buf_len);
+	free(buf);
+	return result;
+}
 
 bool
 backup_file_printf(const char *filename, const char *fmt, ...)
@@ -1380,7 +1384,7 @@ backup_files(const char *from, bool prep_mode)
 	return(ret);
 }
 
-void backup_fix_ddl(void);
+void backup_fix_ddl(CorruptedPages &);
 
 lsn_t get_current_lsn(MYSQL *connection)
 {
@@ -1405,7 +1409,7 @@ lsn_t get_current_lsn(MYSQL *connection)
 lsn_t server_lsn_after_lock;
 extern void backup_wait_for_lsn(lsn_t lsn);
 /** Start --backup */
-bool backup_start()
+bool backup_start(CorruptedPages &corrupted_pages)
 {
 	if (!opt_no_lock) {
 		if (opt_safe_slave_backup) {
@@ -1440,7 +1444,7 @@ bool backup_start()
 
 	msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock);
 	backup_wait_for_lsn(server_lsn_after_lock);
-	backup_fix_ddl();
+	backup_fix_ddl(corrupted_pages);
 
 	// There is no need to stop slave thread before coping non-Innodb data when
 	// --no-lock option is used because --no-lock option requires that no DDL or

extra/mariabackup/backup_copy.h

@@ -33,7 +33,7 @@ copy_file(ds_ctxt_t *datasink,
 	  uint thread_n);
 
 /** Start --backup */
-bool backup_start();
+bool backup_start(CorruptedPages &corrupted_pages);
 /** Release resources after backup_start() */
 void backup_release();
 /** Finish after backup_start() and backup_release() */
@@ -51,5 +51,6 @@ directory_exists(const char *dir, bool create);
 
 lsn_t
 get_current_lsn(MYSQL *connection);
+bool backup_file_print_buf(const char *filename, const char *buf, int buf_len);
 
 #endif

extra/mariabackup/backup_debug.h

@@ -0,0 +1,32 @@
+#pragma once
+#include "my_dbug.h"
+#ifndef DBUG_OFF
+extern char *dbug_mariabackup_get_val(const char *event, const char *key);
+/*
+In debug mode,  execute SQL statement that was passed via environment.
+To use this facility, you need to
+
+1. Add code DBUG_EXECUTE_MARIABACKUP_EVENT("my_event_name", key););
+  to the code. key is usually a table name
+2. Set environment variable my_event_name_$key SQL statement you want to execute
+   when event occurs, in DBUG_EXECUTE_IF from above.
+   In mtr , you can set environment via 'let' statement (do not use $ as the first char
+   for the variable)
+3. start mariabackup with --dbug=+d,debug_mariabackup_events
+*/
+extern void dbug_mariabackup_event(
+	const char *event,const char *key);
+#define DBUG_MARIABACKUP_EVENT(A, B) \
+	DBUG_EXECUTE_IF("mariabackup_events", \
+		dbug_mariabackup_event(A,B););
+#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE) \
+	DBUG_EXECUTE_IF("mariabackup_inject_code", {\
+		char *dbug_val = dbug_mariabackup_get_val(EVENT, KEY); \
+		if (dbug_val && *dbug_val) CODE \
+	})
+#else
+#define DBUG_MARIABACKUP_EVENT(A,B)
+#define DBUG_MARIABACKUP_EVENT_LOCK(A,B)
+#define DBUG_EXECUTE_FOR_KEY(EVENT, KEY, CODE)
+#endif
+

extra/mariabackup/encryption_plugin.cc

@@ -18,7 +18,6 @@
 #include <mysql.h>
 #include <xtrabackup.h>
 #include <encryption_plugin.h>
-#include <backup_copy.h>
 #include <sql_plugin.h>
 #include <sstream>
 #include <vector>

extra/mariabackup/fil_cur.cc

@@ -35,6 +35,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 #include "common.h"
 #include "read_filt.h"
 #include "xtrabackup.h"
+#include "backup_debug.h"
 
 /* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */
 #define XB_FIL_CUR_PAGES 640
@@ -351,19 +352,18 @@ static bool page_is_corrupted(const byte *page, ulint page_no,
 	return buf_page_is_corrupted(true, page, space->flags);
 }
 
-/************************************************************************
-Reads and verifies the next block of pages from the source
+/** Reads and verifies the next block of pages from the source
 file. Positions the cursor after the last read non-corrupted page.
-
+@param[in,out] cursor source file cursor
+@param[out] corrupted_pages adds corrupted pages if
+opt_log_innodb_page_corruption is set
 @return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF
 if there are no more pages to read and XB_FIL_CUR_ERROR on error. */
-xb_fil_cur_result_t
-xb_fil_cur_read(
-/*============*/
-	xb_fil_cur_t*	cursor)	/*!< in/out: source file cursor */
+xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t*	cursor,
+                                    CorruptedPages &corrupted_pages)
 {
 	byte*			page;
-	ulint			i;
+	unsigned			i;
 	ulint			npages;
 	ulint			retry_count;
 	xb_fil_cur_result_t	ret;
@@ -417,7 +417,7 @@ xb_fil_cur_read(
 	cursor->buf_read = 0;
 	cursor->buf_npages = 0;
 	cursor->buf_offset = offset;
-	cursor->buf_page_no = (ulint)(offset / page_size);
+	cursor->buf_page_no = static_cast<unsigned>(offset / page_size);
 
 	if (os_file_read(IORequestRead, cursor->file, cursor->buf, offset,
 			  (ulint) to_read) != DB_SUCCESS) {
@@ -428,26 +428,47 @@ xb_fil_cur_read(
 	partially written pages */
 	for (page = cursor->buf, i = 0; i < npages;
 	     page += page_size, i++) {
-		ulint page_no = cursor->buf_page_no + i;
+		unsigned page_no = cursor->buf_page_no + i;
 
 		if (page_is_corrupted(page, page_no, cursor, space)){
 			retry_count--;
 
 			if (retry_count == 0) {
+				const char *ignore_corruption_warn = opt_log_innodb_page_corruption ?
+					" WARNING!!! The corruption is ignored due to"
+					" log-innodb-page-corruption option, the backup can contain"
+					" corrupted data." : "";
 				msg(cursor->thread_n,
 				    "Error: failed to read page after "
 				    "10 retries. File %s seems to be "
-				    "corrupted.", cursor->abs_path);
-				ret = XB_FIL_CUR_ERROR;
+				    "corrupted.%s", cursor->abs_path, ignore_corruption_warn);
 				ut_print_buf(stderr, page, page_size);
-				break;
+				if (opt_log_innodb_page_corruption) {
+					corrupted_pages.add_page(cursor->node->name, cursor->node->space->id,
+						page_no);
+					retry_count = 1;
+				}
+				else {
+					ret = XB_FIL_CUR_ERROR;
+					break;
+				}
+			}
+			else {
+				msg(cursor->thread_n, "Database page corruption detected at page "
+				    UINT32PF ", retrying...",
+				    page_no);
+				os_thread_sleep(100000);
+				goto read_retry;
 			}
-			msg(cursor->thread_n, "Database page corruption detected at page "
-			    ULINTPF ", retrying...", 
-			    page_no);
-			os_thread_sleep(100000);
-			goto read_retry;
 		}
+		DBUG_EXECUTE_FOR_KEY("add_corrupted_page_for", cursor->node->space->name,
+			{
+				unsigned corrupted_page_no =
+					static_cast<unsigned>(strtoul(dbug_val, NULL, 10));
+				if (page_no == corrupted_page_no)
+					corrupted_pages.add_page(cursor->node->name, cursor->node->space->id,
+						corrupted_page_no);
+			});
 		cursor->buf_read += page_size;
 		cursor->buf_npages++;
 	}

extra/mariabackup/fil_cur.h

@@ -29,6 +29,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 #include "read_filt.h"
 #include "srv0start.h"
 #include "srv0srv.h"
+#include "xtrabackup.h"
 
 struct xb_fil_cur_t {
 	pfs_os_file_t	file;		/*!< source file handle */
@@ -52,7 +53,7 @@ struct xb_fil_cur_t {
 					last cursor read */
 	ib_int64_t	buf_offset;	/*!< file offset of the first page in
 					buffer */
-	ulint		buf_page_no;	/*!< number of the first page in
+	unsigned		buf_page_no;	/*!< number of the first page in
 					buffer */
 	uint		thread_n;	/*!< thread number for diagnostics */
 	ulint		space_id;	/*!< ID of tablespace */
@@ -88,17 +89,15 @@ xb_fil_cur_open(
 	uint		thread_n,	/*!< thread number for diagnostics */
 	ulonglong max_file_size = ULLONG_MAX);
 
-/************************************************************************
-Reads and verifies the next block of pages from the source
+/** Reads and verifies the next block of pages from the source
 file. Positions the cursor after the last read non-corrupted page.
-
+@param[in,out] cursor source file cursor
+@param[out] corrupted_pages adds corrupted pages if
+opt_log_innodb_page_corruption is set
 @return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF
 if there are no more pages to read and XB_FIL_CUR_ERROR on error. */
-xb_fil_cur_result_t
-xb_fil_cur_read(
-/*============*/
-	xb_fil_cur_t*	cursor);	/*!< in/out: source file cursor */
-
+xb_fil_cur_result_t xb_fil_cur_read(xb_fil_cur_t *cursor,
+                                    CorruptedPages &corrupted_pages);
 /************************************************************************
 Close the source file cursor opened with xb_fil_cur_open() and its
 associated read filter. */

extra/mariabackup/write_filt.cc

@@ -32,7 +32,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 /************************************************************************
 Write-through page write filter. */
 static my_bool wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
-			  xb_fil_cur_t *cursor);
+			  xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages);
 static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile);
 
 xb_write_filt_t wf_write_through = {
@@ -45,7 +45,7 @@ xb_write_filt_t wf_write_through = {
 /************************************************************************
 Incremental page write filter. */
 static my_bool wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
-				   xb_fil_cur_t *cursor);
+				   xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages);
 static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt,
 				      ds_file_t *dstfile);
 static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt,
@@ -65,11 +65,11 @@ Initialize incremental page write filter.
 @return TRUE on success, FALSE on error. */
 static my_bool
 wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
-		    xb_fil_cur_t *cursor)
+		    xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages)
 {
 	char				meta_name[FN_REFLEN];
 	xb_wf_incremental_ctxt_t	*cp =
-		&(ctxt->u.wf_incremental_ctxt);
+		&(ctxt->wf_incremental_ctxt);
 
 	ctxt->cursor = cursor;
 
@@ -100,7 +100,9 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
 	strcat(dst_name, ".delta");
 
 	mach_write_to_4(cp->delta_buf, 0x78747261UL); /*"xtra"*/
+
 	cp->npages = 1;
+	cp->corrupted_pages = corrupted_pages;
 
 	return(TRUE);
 }
@@ -112,19 +114,20 @@ Run the next batch of pages through incremental page write filter.
 static my_bool
 wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
 {
-	ulint				i;
+	unsigned				i;
 	xb_fil_cur_t			*cursor = ctxt->cursor;
 	byte				*page;
 	const ulint			page_size = cursor->page_size;
-	xb_wf_incremental_ctxt_t	*cp = &(ctxt->u.wf_incremental_ctxt);
+	xb_wf_incremental_ctxt_t	*cp = &(ctxt->wf_incremental_ctxt);
 
 	for (i = 0, page = cursor->buf; i < cursor->buf_npages;
 	     i++, page += page_size) {
 
-		if (incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN)) {
-
+		if ((!cp->corrupted_pages ||
+				!cp->corrupted_pages->contains(cursor->node->space->id,
+					cursor->buf_page_no + i)) &&
+				incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN))
 			continue;
-		}
 
 		/* updated page */
 		if (cp->npages == page_size / 4) {
@@ -161,7 +164,7 @@ wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
 {
 	xb_fil_cur_t			*cursor = ctxt->cursor;
 	const ulint			page_size = cursor->page_size;
-	xb_wf_incremental_ctxt_t	*cp = &(ctxt->u.wf_incremental_ctxt);
+	xb_wf_incremental_ctxt_t	*cp = &(ctxt->wf_incremental_ctxt);
 
 	if (cp->npages != page_size / 4) {
 		mach_write_to_4(cp->delta_buf + cp->npages * 4, 0xFFFFFFFFUL);
@@ -183,7 +186,7 @@ Free the incremental page write filter's buffer. */
 static void
 wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt)
 {
-	xb_wf_incremental_ctxt_t	*cp = &(ctxt->u.wf_incremental_ctxt);
+	xb_wf_incremental_ctxt_t	*cp = &(ctxt->wf_incremental_ctxt);
 	my_large_free(cp->delta_buf, cp->delta_buf_size);
 }
 
@@ -193,7 +196,7 @@ Initialize the write-through page write filter.
 @return TRUE on success, FALSE on error. */
 static my_bool
 wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)),
-	   xb_fil_cur_t *cursor)
+	   xb_fil_cur_t *cursor, CorruptedPages *)
 {
 	ctxt->cursor = cursor;