diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 4e28e946c2043..1465865b06ce0 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -349,6 +349,9 @@ char orig_argv1[FN_REFLEN]; pthread_mutex_t backup_mutex; pthread_cond_t scanned_lsn_cond; +/** Store the deferred tablespace name during --backup */ +static std::set defer_space_names; + typedef std::map space_id_to_name_t; struct ddl_tracker_t { @@ -358,10 +361,54 @@ struct ddl_tracker_t { std::set drops; /* For DDL operation found in redo log, */ space_id_to_name_t id_to_name; + /** Deferred tablespaces with their ID and name which was + found in redo log of DDL operations */ + space_id_to_name_t deferred_tables; + + /** Insert the deferred tablespace id with the name */ + void insert_defer_id(space_id_t space_id, std::string name) + { + auto it= defer_space_names.find(name); + if (it != defer_space_names.end()) + { + deferred_tables[space_id]= name; + defer_space_names.erase(it); + } + } + + /** Rename the deferred tablespace with new name */ + void rename_defer(space_id_t space_id, std::string old_name, + std::string new_name) + { + if (deferred_tables.find(space_id) != deferred_tables.end()) + deferred_tables[space_id] = new_name; + auto defer_end= defer_space_names.end(); + auto defer= defer_space_names.find(old_name); + if (defer == defer_end) + defer= defer_space_names.find(new_name); + + if (defer != defer_end) + { + deferred_tables[space_id]= new_name; + defer_space_names.erase(defer); + } + } + + /** Delete the deferred tablespace */ + void delete_defer(space_id_t space_id, std::string name) + { + deferred_tables.erase(space_id); + defer_space_names.erase(name); + } }; static ddl_tracker_t ddl_tracker; +/** Stores the space ids of page0 INIT_PAGE redo records. It is +used to indicate whether the given deferred tablespace can +be reconstructed. */ +static std::set first_page_init_ids; + // Convert non-null terminated filename to space name static std::string filename_to_spacename(const void *filename, size_t len); @@ -769,33 +816,57 @@ static std::string filename_to_spacename(const void *filename, size_t len) /** Report an operation to create, delete, or rename a file during backup. @param[in] space_id tablespace identifier -@param[in] create whether the file is being created +@param[in] type redo log file operation type @param[in] name file name (not NUL-terminated) @param[in] len length of name, in bytes @param[in] new_name new file name (NULL if not rename) @param[in] new_len length of new_name, in bytes (0 if NULL) */ -static void backup_file_op(ulint space_id, bool create, +static void backup_file_op(ulint space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len) { - ut_ad(!create || !new_name); ut_ad(name); ut_ad(len); ut_ad(!new_name == !new_len); pthread_mutex_lock(&backup_mutex); - if (create) { - ddl_tracker.id_to_name[space_id] = filename_to_spacename(name, len); + switch(type) { + case FILE_CREATE: + { + std::string space_name = filename_to_spacename(name, len); + ddl_tracker.id_to_name[space_id] = space_name; + ddl_tracker.delete_defer(space_id, space_name); msg("DDL tracking : create %zu \"%.*s\"", space_id, int(len), name); } - else if (new_name) { - ddl_tracker.id_to_name[space_id] = filename_to_spacename(new_name, new_len); + break; + case FILE_MODIFY: + ddl_tracker.insert_defer_id( + space_id, filename_to_spacename(name, len)); + msg("DDL tracking : modify %zu \"%.*s\"", space_id, int(len), name); + break; + case FILE_RENAME: + { + std::string new_space_name = filename_to_spacename( + new_name, new_len); + std::string old_space_name = filename_to_spacename( + name, len); + ddl_tracker.id_to_name[space_id] = new_space_name; + ddl_tracker.rename_defer(space_id, old_space_name, + new_space_name); msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"", space_id, int(len), name, int(new_len), new_name); - } else { + } + break; + case FILE_DELETE: ddl_tracker.drops.insert(space_id); + ddl_tracker.delete_defer( + space_id, filename_to_spacename(name, len)); msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name); + break; + default: + ut_ad(0); + break; } pthread_mutex_unlock(&backup_mutex); } @@ -810,29 +881,38 @@ static void backup_file_op(ulint space_id, bool create, We will abort backup in this case. */ -static void backup_file_op_fail(ulint space_id, bool create, +static void backup_file_op_fail(ulint space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len) { - bool fail; - if (create) { - msg("DDL tracking : create %zu \"%.*s\"", - space_id, int(len), name); - std::string spacename = filename_to_spacename(name, len); - fail = !check_if_skip_table(spacename.c_str()); - } - else if (new_name) { + bool fail = false; + switch(type) { + case FILE_CREATE: + msg("DDL tracking : create %zu \"%.*s\"", space_id, int(len), name); + fail = !check_if_skip_table( + filename_to_spacename(name, len).c_str()); + break; + case FILE_MODIFY: + msg("DDL tracking : modify %zu \"%.*s\"", space_id, int(len), name); + break; + case FILE_RENAME: msg("DDL tracking : rename %zu \"%.*s\",\"%.*s\"", space_id, int(len), name, int(new_len), new_name); - std::string spacename = filename_to_spacename(name, len); - std::string new_spacename = filename_to_spacename(new_name, new_len); - fail = !check_if_skip_table(spacename.c_str()) || !check_if_skip_table(new_spacename.c_str()); - } - else { - std::string spacename = filename_to_spacename(name, len); - fail = !check_if_skip_table(spacename.c_str()); + fail = !check_if_skip_table( + filename_to_spacename(name, len).c_str()) + || !check_if_skip_table( + filename_to_spacename(new_name, new_len).c_str()); + break; + case FILE_DELETE: + fail = !check_if_skip_table( + filename_to_spacename(name, len).c_str()); msg("DDL tracking : delete %zu \"%.*s\"", space_id, int(len), name); + break; + default: + ut_ad(0); + break; } + if (fail) { ut_a(opt_no_lock); die("DDL operation detected in the late phase of backup." @@ -840,6 +920,12 @@ static void backup_file_op_fail(ulint space_id, bool create, } } +/* Function to store the space id of page0 INIT_PAGE +@param space_id space id which has page0 init page */ +static void backup_first_page_op(ulint space_id) +{ + first_page_init_ids.insert(space_id); +} /* Retrieve default data directory, to be used with --copy-back. @@ -3234,7 +3320,8 @@ static void xb_load_single_table_tablespace(const char *dirname, { ut_ad(srv_operation == SRV_OPERATION_BACKUP || srv_operation == SRV_OPERATION_RESTORE_DELTA - || srv_operation == SRV_OPERATION_RESTORE); + || srv_operation == SRV_OPERATION_RESTORE + || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER); /* Ignore .isl files on XtraBackup recovery. All tablespaces must be local. */ if (is_remote && srv_operation == SRV_OPERATION_RESTORE_DELTA) { @@ -3303,6 +3390,10 @@ static void xb_load_single_table_tablespace(const char *dirname, } if (!defer && file->m_defer) { + const char *file_path = file->filepath(); + defer_space_names.insert( + filename_to_spacename( + file_path, strlen(file_path))); delete file; ut_free(name); return; @@ -4417,6 +4508,7 @@ static bool xtrabackup_backup_func() srv_operation = SRV_OPERATION_BACKUP; log_file_op = backup_file_op; + first_page_init = backup_first_page_op; metadata_to_lsn = 0; /* initialize components */ @@ -4431,6 +4523,7 @@ static bool xtrabackup_backup_func() } log_file_op = NULL; + first_page_init = NULL; if (dst_log_file) { ds_close(dst_log_file); dst_log_file = NULL; @@ -4733,6 +4826,7 @@ static bool xtrabackup_backup_func() innodb_shutdown(); log_file_op = NULL; + first_page_init = NULL; pthread_mutex_destroy(&backup_mutex); pthread_cond_destroy(&scanned_lsn_cond); if (!corrupted_pages.empty()) { @@ -4814,7 +4908,9 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) continue; } - if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end()) { + if (ddl_tracker.drops.find(id) == ddl_tracker.drops.end() + && ddl_tracker.deferred_tables.find(id) + == ddl_tracker.deferred_tables.end()) { dropped_tables.erase(name); new_tables[id] = name; if (opt_log_innodb_page_corruption) @@ -4859,6 +4955,41 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) DBUG_EXECUTE_IF("check_mdl_lock_works", DBUG_ASSERT(new_tables.size() == 0);); + srv_operation = SRV_OPERATION_BACKUP_NO_DEFER; + + /* Mariabackup detected the FILE_MODIFY or FILE_RENAME + for the deferred tablespace. So it needs to read the + tablespace again if innodb doesn't have page0 initialization + redo log for it */ + for (space_id_to_name_t::iterator iter = + ddl_tracker.deferred_tables.begin(); + iter != ddl_tracker.deferred_tables.end(); + iter++) { + if (check_if_skip_table(iter->second.c_str())) { + continue; + } + + if (first_page_init_ids.find(iter->first) + != first_page_init_ids.end()) { + new_tables[iter->first] = iter->second.c_str(); + continue; + } + + xb_load_single_table_tablespace(iter->second, false); + } + + /* Mariabackup doesn't detect any FILE_OP for the deferred + tablespace. There is a possiblity that page0 could've + been corrupted persistently in the disk */ + for (auto space_name: defer_space_names) { + if (!check_if_skip_table(space_name.c_str())) { + xb_load_single_table_tablespace( + space_name, false); + } + } + + srv_operation = SRV_OPERATION_BACKUP; + for (const auto &t : new_tables) { if (!check_if_skip_table(t.second.c_str())) { xb_load_single_table_tablespace(t.second, false, diff --git a/mysql-test/suite/mariabackup/defer_space.result b/mysql-test/suite/mariabackup/defer_space.result new file mode 100644 index 0000000000000..41239c476e751 --- /dev/null +++ b/mysql-test/suite/mariabackup/defer_space.result @@ -0,0 +1,26 @@ +call mtr.add_suppression("InnoDB: Expected tablespace id .*"); +# Mariabackup --backup with page0 INIT_PAGE redo record +# and there is no FILE_CREATE for the tablespace t1 +SET DEBUG_DBUG="+d,checkpoint_after_file_create"; +CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 VALUES(1); +# xtrabackup backup +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart +SELECT * FROM t1; +f1 +1 +DROP TABLE t1; +SET DEBUG_DBUG="-d,checkpoint_after_file_create"; +# Mariabackup fails after corrupting the page0 in disk +# and there is no INIT_PAGE for page0 +CREATE TABLE t1(c INT) ENGINE=INNODB; +# Corrupt the table +# restart +# xtrabackup backup +FOUND 10 /Header page consists of zero bytes*/ in backup.log +UNLOCK TABLES; +DROP TABLE t1; diff --git a/mysql-test/suite/mariabackup/defer_space.test b/mysql-test/suite/mariabackup/defer_space.test new file mode 100644 index 0000000000000..65f2262fd1dee --- /dev/null +++ b/mysql-test/suite/mariabackup/defer_space.test @@ -0,0 +1,69 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: Expected tablespace id .*"); +--echo # Mariabackup --backup with page0 INIT_PAGE redo record +--echo # and there is no FILE_CREATE for the tablespace t1 +SET DEBUG_DBUG="+d,checkpoint_after_file_create"; +CREATE TABLE t1(f1 INT NOT NULL)ENGINE=InnoDB; +INSERT INTO t1 VALUES(1); + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; + +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log + +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +-- source include/restart_and_restore.inc +--enable_result_log + +SELECT * FROM t1; +DROP TABLE t1; +rmdir $targetdir; +SET DEBUG_DBUG="-d,checkpoint_after_file_create"; + +--echo # Mariabackup fails after corrupting the page0 in disk +--echo # and there is no INIT_PAGE for page0 + +CREATE TABLE t1(c INT) ENGINE=INNODB; +let MYSQLD_DATADIR=`select @@datadir`; +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +--source include/shutdown_mysqld.inc + +--echo # Corrupt the table + +perl; +use strict; +use warnings; +use Fcntl qw(:DEFAULT :seek); +my $page_size = $ENV{INNODB_PAGE_SIZE}; + +sysopen FILE, "$ENV{MYSQLD_DATADIR}/test/t1.ibd", O_RDWR +|| die "Cannot open t1.ibd\n"; +sysseek(FILE, 0, SEEK_SET) || die "Cannot seek t1.ibd\n"; +my $page=chr(0) x $page_size; +syswrite(FILE, $page, $page_size)==$page_size; +close FILE or die "close"; +EOF + +--source include/start_mysqld.inc +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log; +--disable_result_log +--error 1 +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --core-file > $backuplog; +--enable_result_log + +--let SEARCH_PATTERN=Header page consists of zero bytes* +--let SEARCH_FILE=$backuplog +--source include/search_pattern_in_file.inc +UNLOCK TABLES; +DROP TABLE t1; +rmdir $targetdir; +remove_file $backuplog; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 681cee32fd581..764ee73f8144f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2088,6 +2088,9 @@ fil_ibd_create( } } + DBUG_EXECUTE_IF("checkpoint_after_file_create", + log_make_checkpoint();); + if (fil_space_t* space = fil_space_t::create(space_id, flags, FIL_TYPE_TABLESPACE, crypt_data, mode)) { @@ -2432,6 +2435,7 @@ fil_ibd_discover( switch (srv_operation) { case SRV_OPERATION_BACKUP: case SRV_OPERATION_RESTORE_DELTA: + case SRV_OPERATION_BACKUP_NO_DEFER: ut_ad(0); break; case SRV_OPERATION_RESTORE_EXPORT: diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 838e7357a6a5c..9bca993b0f7f6 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -82,15 +82,19 @@ void recv_sys_justify_left_parsing_buf(); /** Report an operation to create, delete, or rename a file during backup. @param[in] space_id tablespace identifier -@param[in] create whether the file is being created +@param[in] type file operation redo log type @param[in] name file name (not NUL-terminated) @param[in] len length of name, in bytes @param[in] new_name new file name (NULL if not rename) @param[in] new_len length of new_name, in bytes (0 if NULL) */ -extern void (*log_file_op)(ulint space_id, bool create, +extern void (*log_file_op)(ulint space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len); +/** Report an operation which does INIT_PAGE for page0 during backup. +@param space_id tablespace identifier */ +extern void (*first_page_init)(ulint space_id); + /** Stored redo log record */ struct log_rec_t { diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 30c957c83cb51..d31f074486ab3 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -419,7 +419,10 @@ enum srv_operation_mode { /** Mariabackup restoring the incremental part of a backup */ SRV_OPERATION_RESTORE_DELTA, /** Mariabackup restoring a backup for subsequent --export */ - SRV_OPERATION_RESTORE_EXPORT + SRV_OPERATION_RESTORE_EXPORT, + /** Mariabackup taking a backup and avoid deferring + any tablespace */ + SRV_OPERATION_BACKUP_NO_DEFER }; /** Current mode of operation */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 71b9a03448285..8074ef2656c4f 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -939,15 +939,17 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p, /** Report an operation to create, delete, or rename a file during backup. @param[in] space_id tablespace identifier -@param[in] create whether the file is being created +@param[in] type redo log type @param[in] name file name (not NUL-terminated) @param[in] len length of name, in bytes @param[in] new_name new file name (NULL if not rename) @param[in] new_len length of new_name, in bytes (0 if NULL) */ -void (*log_file_op)(ulint space_id, bool create, +void (*log_file_op)(ulint space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len); +void (*first_page_init)(ulint space_id); + /** Information about initializing page contents during redo log processing. FIXME: Rely on recv_sys.pages! */ class mlog_init_t @@ -1151,7 +1153,8 @@ void fil_name_process(char* name, ulint len, ulint space_id, bool deleted, lsn_t lsn, store_t *store) { - if (srv_operation == SRV_OPERATION_BACKUP) { + if (srv_operation == SRV_OPERATION_BACKUP + || srv_operation == SRV_OPERATION_BACKUP_NO_DEFER) { return; } @@ -2100,6 +2103,8 @@ static void store_freed_or_init_rec(page_id_t page_id, bool freed) { uint32_t space_id= page_id.space(); uint32_t page_no= page_id.page_no(); + if (!freed && page_no == 0 && first_page_init) + first_page_init(space_id); if (is_predefined_tablespace(space_id)) { if (!srv_immediate_scrub_data_uncompressed) @@ -2575,8 +2580,8 @@ bool recv_sys_t::parse(lsn_t checkpoint_lsn, store_t *store, bool apply) if (fn2) fil_name_process(const_cast(fn2), fn2end - fn2, space_id, false, start_lsn, store); - if ((b & 0xf0) < FILE_MODIFY && log_file_op) - log_file_op(space_id, (b & 0xf0) == FILE_CREATE, + if ((b & 0xf0) < FILE_CHECKPOINT && log_file_op) + log_file_op(space_id, b & 0xf0, l, static_cast(fnend - fn), reinterpret_cast(fn2), fn2 ? static_cast(fn2end - fn2) : 0); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index f379124b79414..830ccbcaa5639 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1473,6 +1473,7 @@ dberr_t srv_start(bool create_new_db) break; case SRV_OPERATION_RESTORE_DELTA: case SRV_OPERATION_BACKUP: + case SRV_OPERATION_BACKUP_NO_DEFER: ut_ad("wrong mariabackup mode" == 0); } @@ -1947,6 +1948,7 @@ void innodb_shutdown() switch (srv_operation) { case SRV_OPERATION_BACKUP: case SRV_OPERATION_RESTORE_DELTA: + case SRV_OPERATION_BACKUP_NO_DEFER: break; case SRV_OPERATION_RESTORE: case SRV_OPERATION_RESTORE_EXPORT: